{ "schemaVersion": 1, "publicationId": "2026-10-27-m5max-native-0-1-54", "interpretation": "All attempts included regardless of success/eligibility. Actual in-trial engine-log rates, not low-token preflight probes. MLX decode is a median of pulse rates; llama.cpp is generated tokens divided by decode time. Recovered MLX observed prefill is a median of rounded first-token chat-event rates within each trial, not aggregate request throughput. Exact MLX prefill duration and granted context remain unknown. macOS GPU utilization/memory are unavailable, not zero. System RAM is machine-wide; process RSS is sampled. First artifact records first write, not completed output.", "coverage": { "duration": { "available": 516, "total": 516, "unknownCells": [] }, "firstToken": { "available": 516, "total": 516, "unknownCells": [] }, "firstAction": { "available": 516, "total": 516, "unknownCells": [] }, "firstArtifact": { "available": 494, "total": 516, "unknownCells": [ "r1-qwen-llamacpp-developer-dev-craftbook-routing", "r1-gemma-llamacpp-developer-dev-craftbook-routing", "r1-gemma-mlx-complex-work-craftbook-find-vs-create", "r1-qwen-llamacpp-complex-work-craftbook-find-vs-create", "r1-qwen-mlx-complex-work-craftbook-author-gate-script", "r2-gemma-mlx-developer-dev-craftbook-routing", "r2-qwen-mlx-developer-dev-craftbook-routing", "r2-qwen-llamacpp-developer-dev-craftbook-routing", "r2-qwen-mlx-complex-work-craftbook-find-vs-create", "r2-qwen-llamacpp-complex-work-craftbook-find-vs-create", "r2-gemma-llamacpp-complex-work-craftbook-author-params", "r2-qwen-mlx-complex-work-craftbook-author-gate-script", "r2-gemma-llamacpp-complex-work-craftbook-author-fanout", "r3-qwen-mlx-developer-dev-craftbook-routing", "r3-gemma-llamacpp-developer-craftbook-code-review", "r3-qwen-mlx-complex-work-craftbook-find-vs-create", "r3-gemma-mlx-complex-work-craftbook-find-vs-create", "r3-gemma-llamacpp-complex-work-craftbook-find-vs-create", "r3-gemma-mlx-complex-work-craftbook-export-generalize", "r3-qwen-mlx-complex-work-craftbook-author-gate-script", "r3-qwen-llamacpp-complex-work-craftbook-author-gate-script", "r3-qwen-mlx-complex-work-craftbook-author-fanout" ] }, "decodeThroughput": { "available": 514, "total": 516, "unknownCells": [ "r1-qwen-mlx-productivity-constrained-comms", "r3-qwen-mlx-productivity-constrained-comms" ] }, "prefillThroughput": { "available": 258, "total": 516, "unknownCells": [ "r1-qwen-mlx-core-tictactoe", "r1-gemma-mlx-core-tictactoe", "r1-gemma-mlx-core-petshop", "r1-qwen-mlx-core-petshop", "r1-gemma-mlx-core-tankcombat", "r1-qwen-mlx-core-tankcombat", "r1-qwen-mlx-core-schema-migration", "r1-gemma-mlx-core-schema-migration", "r1-qwen-mlx-core-failing-tests-spec", "r1-gemma-mlx-core-failing-tests-spec", "r1-gemma-mlx-core-symptom-debug", "r1-qwen-mlx-core-symptom-debug", "r1-gemma-mlx-core-data-wrangle", "r1-qwen-mlx-core-data-wrangle", "r1-qwen-mlx-core-incident-postmortem", "r1-gemma-mlx-core-incident-postmortem", "r1-qwen-mlx-core-ops-runbook-anomaly", "r1-gemma-mlx-core-ops-runbook-anomaly", "r1-gemma-mlx-core-plan-and-estimate", "r1-qwen-mlx-core-plan-and-estimate", "r1-gemma-mlx-core-conflict-synthesis", "r1-qwen-mlx-core-conflict-synthesis", "r1-qwen-mlx-productivity-constrained-comms", "r1-gemma-mlx-productivity-constrained-comms", "r1-qwen-mlx-productivity-records-intake", "r1-gemma-mlx-productivity-records-intake", "r1-gemma-mlx-productivity-plan-and-estimate", "r1-qwen-mlx-productivity-plan-and-estimate", "r1-gemma-mlx-productivity-meeting-followup", "r1-qwen-mlx-productivity-meeting-followup", "r1-qwen-mlx-productivity-craftbook-spreadsheet-model", "r1-gemma-mlx-productivity-craftbook-spreadsheet-model", "r1-qwen-mlx-productivity-conflict-synthesis", "r1-gemma-mlx-productivity-conflict-synthesis", "r1-gemma-mlx-productivity-docblocks-theme-roundtrip", "r1-qwen-mlx-productivity-docblocks-theme-roundtrip", "r1-gemma-mlx-productivity-craftbook-powerpoint-deck", "r1-qwen-mlx-productivity-craftbook-powerpoint-deck", "r1-qwen-mlx-productivity-wikipedia-research-brief", "r1-gemma-mlx-productivity-wikipedia-research-brief", "r1-qwen-mlx-productivity-craftbook-week-plan", "r1-gemma-mlx-productivity-craftbook-week-plan", "r1-gemma-mlx-productivity-craftbook-ab-test-readout", "r1-qwen-mlx-productivity-craftbook-ab-test-readout", "r1-gemma-mlx-productivity-craftbook-annotated-bibliography", "r1-qwen-mlx-productivity-craftbook-annotated-bibliography", "r1-qwen-mlx-productivity-craftbook-research-to-document", "r1-gemma-mlx-productivity-craftbook-research-to-document", "r1-qwen-mlx-developer-fix-squisq-bugs", "r1-gemma-mlx-developer-fix-squisq-bugs", "r1-gemma-mlx-developer-dev-craftbook-routing", "r1-qwen-mlx-developer-dev-craftbook-routing", "r1-gemma-mlx-developer-interface-contract", "r1-qwen-mlx-developer-interface-contract", "r1-qwen-mlx-developer-craftbook-code-review", "r1-gemma-mlx-developer-craftbook-code-review", "r1-qwen-mlx-developer-craftbook-api-contract-review", "r1-gemma-mlx-developer-craftbook-api-contract-review", "r1-gemma-mlx-developer-craftbook-deep-security-review", "r1-qwen-mlx-developer-craftbook-deep-security-review", "r1-gemma-mlx-developer-large-pr-review", "r1-qwen-mlx-developer-large-pr-review", "r1-qwen-mlx-developer-craftbook-codemod-sweep", "r1-gemma-mlx-developer-craftbook-codemod-sweep", "r1-qwen-mlx-developer-craftbook-refactor-module", "r1-gemma-mlx-developer-craftbook-refactor-module", "r1-gemma-mlx-developer-craftbook-bug-fix-tdd", "r1-qwen-mlx-developer-craftbook-bug-fix-tdd", "r1-gemma-mlx-complex-work-craftbook-find-vs-create", "r1-qwen-mlx-complex-work-craftbook-find-vs-create", "r1-qwen-mlx-complex-work-craftbook-edit-midtask", "r1-gemma-mlx-complex-work-craftbook-edit-midtask", "r1-qwen-mlx-complex-work-craftbook-invoice-run", "r1-gemma-mlx-complex-work-craftbook-invoice-run", "r1-gemma-mlx-complex-work-craftbook-author-linear", "r1-qwen-mlx-complex-work-craftbook-author-linear", "r1-gemma-mlx-complex-work-craftbook-route-multi", "r1-qwen-mlx-complex-work-craftbook-route-multi", "r1-qwen-mlx-complex-work-craftbook-export-generalize", "r1-gemma-mlx-complex-work-craftbook-export-generalize", "r1-qwen-mlx-complex-work-craftbook-author-params", "r1-gemma-mlx-complex-work-craftbook-author-params", "r1-gemma-mlx-complex-work-craftbook-author-gate-script", "r1-qwen-mlx-complex-work-craftbook-author-gate-script", "r1-gemma-mlx-complex-work-craftbook-author-fanout", "r1-qwen-mlx-complex-work-craftbook-author-fanout", "r2-gemma-mlx-core-tictactoe", "r2-qwen-mlx-core-tictactoe", "r2-gemma-mlx-core-petshop", "r2-qwen-mlx-core-petshop", "r2-qwen-mlx-core-tankcombat", "r2-gemma-mlx-core-tankcombat", "r2-qwen-mlx-core-schema-migration", "r2-gemma-mlx-core-schema-migration", "r2-gemma-mlx-core-failing-tests-spec", "r2-qwen-mlx-core-failing-tests-spec", "r2-gemma-mlx-core-symptom-debug", "r2-qwen-mlx-core-symptom-debug", "r2-qwen-mlx-core-data-wrangle", "r2-gemma-mlx-core-data-wrangle", "r2-qwen-mlx-core-incident-postmortem", "r2-gemma-mlx-core-incident-postmortem", "r2-gemma-mlx-core-ops-runbook-anomaly", "r2-qwen-mlx-core-ops-runbook-anomaly", "r2-gemma-mlx-core-plan-and-estimate", "r2-qwen-mlx-core-plan-and-estimate", "r2-qwen-mlx-core-conflict-synthesis", "r2-gemma-mlx-core-conflict-synthesis", "r2-qwen-mlx-productivity-constrained-comms", "r2-gemma-mlx-productivity-constrained-comms", "r2-gemma-mlx-productivity-records-intake", "r2-qwen-mlx-productivity-records-intake", "r2-gemma-mlx-productivity-plan-and-estimate", "r2-qwen-mlx-productivity-plan-and-estimate", "r2-qwen-mlx-productivity-meeting-followup", "r2-gemma-mlx-productivity-meeting-followup", "r2-qwen-mlx-productivity-craftbook-spreadsheet-model", "r2-gemma-mlx-productivity-craftbook-spreadsheet-model", "r2-gemma-mlx-productivity-conflict-synthesis", "r2-qwen-mlx-productivity-conflict-synthesis", "r2-gemma-mlx-productivity-docblocks-theme-roundtrip", "r2-qwen-mlx-productivity-docblocks-theme-roundtrip", "r2-qwen-mlx-productivity-craftbook-powerpoint-deck", "r2-gemma-mlx-productivity-craftbook-powerpoint-deck", "r2-qwen-mlx-productivity-wikipedia-research-brief", "r2-gemma-mlx-productivity-wikipedia-research-brief", "r2-gemma-mlx-productivity-craftbook-week-plan", "r2-qwen-mlx-productivity-craftbook-week-plan", "r2-gemma-mlx-productivity-craftbook-ab-test-readout", "r2-qwen-mlx-productivity-craftbook-ab-test-readout", "r2-qwen-mlx-productivity-craftbook-annotated-bibliography", "r2-gemma-mlx-productivity-craftbook-annotated-bibliography", "r2-qwen-mlx-productivity-craftbook-research-to-document", "r2-gemma-mlx-productivity-craftbook-research-to-document", "r2-gemma-mlx-developer-fix-squisq-bugs", "r2-qwen-mlx-developer-fix-squisq-bugs", "r2-gemma-mlx-developer-dev-craftbook-routing", "r2-qwen-mlx-developer-dev-craftbook-routing", "r2-qwen-mlx-developer-interface-contract", "r2-gemma-mlx-developer-interface-contract", "r2-qwen-mlx-developer-craftbook-code-review", "r2-gemma-mlx-developer-craftbook-code-review", "r2-gemma-mlx-developer-craftbook-api-contract-review", "r2-qwen-mlx-developer-craftbook-api-contract-review", "r2-gemma-mlx-developer-craftbook-deep-security-review", "r2-qwen-mlx-developer-craftbook-deep-security-review", "r2-qwen-mlx-developer-large-pr-review", "r2-gemma-mlx-developer-large-pr-review", "r2-qwen-mlx-developer-craftbook-codemod-sweep", "r2-gemma-mlx-developer-craftbook-codemod-sweep", "r2-gemma-mlx-developer-craftbook-refactor-module", "r2-qwen-mlx-developer-craftbook-refactor-module", "r2-gemma-mlx-developer-craftbook-bug-fix-tdd", "r2-qwen-mlx-developer-craftbook-bug-fix-tdd", "r2-qwen-mlx-complex-work-craftbook-find-vs-create", "r2-gemma-mlx-complex-work-craftbook-find-vs-create", "r2-qwen-mlx-complex-work-craftbook-edit-midtask", "r2-gemma-mlx-complex-work-craftbook-edit-midtask", "r2-gemma-mlx-complex-work-craftbook-invoice-run", "r2-qwen-mlx-complex-work-craftbook-invoice-run", "r2-gemma-mlx-complex-work-craftbook-author-linear", "r2-qwen-mlx-complex-work-craftbook-author-linear", "r2-qwen-mlx-complex-work-craftbook-route-multi", "r2-gemma-mlx-complex-work-craftbook-route-multi", "r2-qwen-mlx-complex-work-craftbook-export-generalize", "r2-gemma-mlx-complex-work-craftbook-export-generalize", "r2-gemma-mlx-complex-work-craftbook-author-params", "r2-qwen-mlx-complex-work-craftbook-author-params", "r2-gemma-mlx-complex-work-craftbook-author-gate-script", "r2-qwen-mlx-complex-work-craftbook-author-gate-script", "r2-qwen-mlx-complex-work-craftbook-author-fanout", "r2-gemma-mlx-complex-work-craftbook-author-fanout", "r3-gemma-mlx-core-tictactoe", "r3-qwen-mlx-core-tictactoe", "r3-qwen-mlx-core-petshop", "r3-gemma-mlx-core-petshop", "r3-qwen-mlx-core-tankcombat", "r3-gemma-mlx-core-tankcombat", "r3-gemma-mlx-core-schema-migration", "r3-qwen-mlx-core-schema-migration", "r3-gemma-mlx-core-failing-tests-spec", "r3-qwen-mlx-core-failing-tests-spec", "r3-qwen-mlx-core-symptom-debug", "r3-gemma-mlx-core-symptom-debug", "r3-qwen-mlx-core-data-wrangle", "r3-gemma-mlx-core-data-wrangle", "r3-gemma-mlx-core-incident-postmortem", "r3-qwen-mlx-core-incident-postmortem", "r3-gemma-mlx-core-ops-runbook-anomaly", "r3-qwen-mlx-core-ops-runbook-anomaly", "r3-qwen-mlx-core-plan-and-estimate", "r3-gemma-mlx-core-plan-and-estimate", "r3-qwen-mlx-core-conflict-synthesis", "r3-gemma-mlx-core-conflict-synthesis", "r3-gemma-mlx-productivity-constrained-comms", "r3-qwen-mlx-productivity-constrained-comms", "r3-gemma-mlx-productivity-records-intake", "r3-qwen-mlx-productivity-records-intake", "r3-qwen-mlx-productivity-plan-and-estimate", "r3-gemma-mlx-productivity-plan-and-estimate", "r3-qwen-mlx-productivity-meeting-followup", "r3-gemma-mlx-productivity-meeting-followup", "r3-gemma-mlx-productivity-craftbook-spreadsheet-model", "r3-qwen-mlx-productivity-craftbook-spreadsheet-model", "r3-gemma-mlx-productivity-conflict-synthesis", "r3-qwen-mlx-productivity-conflict-synthesis", "r3-qwen-mlx-productivity-docblocks-theme-roundtrip", "r3-gemma-mlx-productivity-docblocks-theme-roundtrip", "r3-qwen-mlx-productivity-craftbook-powerpoint-deck", "r3-gemma-mlx-productivity-craftbook-powerpoint-deck", "r3-gemma-mlx-productivity-wikipedia-research-brief", "r3-qwen-mlx-productivity-wikipedia-research-brief", "r3-gemma-mlx-productivity-craftbook-week-plan", "r3-qwen-mlx-productivity-craftbook-week-plan", "r3-qwen-mlx-productivity-craftbook-ab-test-readout", "r3-gemma-mlx-productivity-craftbook-ab-test-readout", "r3-qwen-mlx-productivity-craftbook-annotated-bibliography", "r3-gemma-mlx-productivity-craftbook-annotated-bibliography", "r3-gemma-mlx-productivity-craftbook-research-to-document", "r3-qwen-mlx-productivity-craftbook-research-to-document", "r3-gemma-mlx-developer-fix-squisq-bugs", "r3-qwen-mlx-developer-fix-squisq-bugs", "r3-qwen-mlx-developer-dev-craftbook-routing", "r3-gemma-mlx-developer-dev-craftbook-routing", "r3-qwen-mlx-developer-interface-contract", "r3-gemma-mlx-developer-interface-contract", "r3-gemma-mlx-developer-craftbook-code-review", "r3-qwen-mlx-developer-craftbook-code-review", "r3-gemma-mlx-developer-craftbook-api-contract-review", "r3-qwen-mlx-developer-craftbook-api-contract-review", "r3-qwen-mlx-developer-craftbook-deep-security-review", "r3-gemma-mlx-developer-craftbook-deep-security-review", "r3-qwen-mlx-developer-large-pr-review", "r3-gemma-mlx-developer-large-pr-review", "r3-gemma-mlx-developer-craftbook-codemod-sweep", "r3-qwen-mlx-developer-craftbook-codemod-sweep", "r3-gemma-mlx-developer-craftbook-refactor-module", "r3-qwen-mlx-developer-craftbook-refactor-module", "r3-qwen-mlx-developer-craftbook-bug-fix-tdd", "r3-gemma-mlx-developer-craftbook-bug-fix-tdd", "r3-qwen-mlx-complex-work-craftbook-find-vs-create", "r3-gemma-mlx-complex-work-craftbook-find-vs-create", "r3-gemma-mlx-complex-work-craftbook-edit-midtask", "r3-qwen-mlx-complex-work-craftbook-edit-midtask", "r3-gemma-mlx-complex-work-craftbook-invoice-run", "r3-qwen-mlx-complex-work-craftbook-invoice-run", "r3-qwen-mlx-complex-work-craftbook-author-linear", "r3-gemma-mlx-complex-work-craftbook-author-linear", "r3-qwen-mlx-complex-work-craftbook-route-multi", "r3-gemma-mlx-complex-work-craftbook-route-multi", "r3-gemma-mlx-complex-work-craftbook-export-generalize", "r3-qwen-mlx-complex-work-craftbook-export-generalize", "r3-gemma-mlx-complex-work-craftbook-author-params", "r3-qwen-mlx-complex-work-craftbook-author-params", "r3-qwen-mlx-complex-work-craftbook-author-gate-script", "r3-gemma-mlx-complex-work-craftbook-author-gate-script", "r3-qwen-mlx-complex-work-craftbook-author-fanout", "r3-gemma-mlx-complex-work-craftbook-author-fanout" ] }, "inputTokens": { "available": 516, "total": 516, "unknownCells": [] }, "outputTokens": { "available": 516, "total": 516, "unknownCells": [] }, "peakRss": { "available": 516, "total": 516, "unknownCells": [] }, "systemMemory": { "available": 516, "total": 516, "unknownCells": [] }, "gpuUtilization": { "available": 0, "total": 516, "unknownCells": [ "r1-qwen-mlx-core-tictactoe", "r1-qwen-llamacpp-core-tictactoe", "r1-gemma-mlx-core-tictactoe", "r1-gemma-llamacpp-core-tictactoe", "r1-qwen-llamacpp-core-petshop", "r1-gemma-mlx-core-petshop", "r1-gemma-llamacpp-core-petshop", "r1-qwen-mlx-core-petshop", "r1-gemma-mlx-core-tankcombat", "r1-gemma-llamacpp-core-tankcombat", "r1-qwen-mlx-core-tankcombat", "r1-qwen-llamacpp-core-tankcombat", "r1-gemma-llamacpp-core-schema-migration", "r1-qwen-mlx-core-schema-migration", "r1-qwen-llamacpp-core-schema-migration", "r1-gemma-mlx-core-schema-migration", "r1-qwen-mlx-core-failing-tests-spec", "r1-qwen-llamacpp-core-failing-tests-spec", "r1-gemma-mlx-core-failing-tests-spec", "r1-gemma-llamacpp-core-failing-tests-spec", "r1-qwen-llamacpp-core-symptom-debug", "r1-gemma-mlx-core-symptom-debug", "r1-gemma-llamacpp-core-symptom-debug", "r1-qwen-mlx-core-symptom-debug", "r1-gemma-mlx-core-data-wrangle", "r1-gemma-llamacpp-core-data-wrangle", "r1-qwen-mlx-core-data-wrangle", "r1-qwen-llamacpp-core-data-wrangle", "r1-gemma-llamacpp-core-incident-postmortem", "r1-qwen-mlx-core-incident-postmortem", "r1-qwen-llamacpp-core-incident-postmortem", "r1-gemma-mlx-core-incident-postmortem", "r1-qwen-mlx-core-ops-runbook-anomaly", "r1-qwen-llamacpp-core-ops-runbook-anomaly", "r1-gemma-mlx-core-ops-runbook-anomaly", "r1-gemma-llamacpp-core-ops-runbook-anomaly", "r1-qwen-llamacpp-core-plan-and-estimate", "r1-gemma-mlx-core-plan-and-estimate", "r1-gemma-llamacpp-core-plan-and-estimate", "r1-qwen-mlx-core-plan-and-estimate", "r1-gemma-mlx-core-conflict-synthesis", "r1-gemma-llamacpp-core-conflict-synthesis", "r1-qwen-mlx-core-conflict-synthesis", "r1-qwen-llamacpp-core-conflict-synthesis", "r1-gemma-llamacpp-productivity-constrained-comms", "r1-qwen-mlx-productivity-constrained-comms", "r1-qwen-llamacpp-productivity-constrained-comms", "r1-gemma-mlx-productivity-constrained-comms", "r1-qwen-mlx-productivity-records-intake", "r1-qwen-llamacpp-productivity-records-intake", "r1-gemma-mlx-productivity-records-intake", "r1-gemma-llamacpp-productivity-records-intake", "r1-qwen-llamacpp-productivity-plan-and-estimate", "r1-gemma-mlx-productivity-plan-and-estimate", "r1-gemma-llamacpp-productivity-plan-and-estimate", "r1-qwen-mlx-productivity-plan-and-estimate", "r1-gemma-mlx-productivity-meeting-followup", "r1-gemma-llamacpp-productivity-meeting-followup", "r1-qwen-mlx-productivity-meeting-followup", "r1-qwen-llamacpp-productivity-meeting-followup", "r1-gemma-llamacpp-productivity-craftbook-spreadsheet-model", "r1-qwen-mlx-productivity-craftbook-spreadsheet-model", "r1-qwen-llamacpp-productivity-craftbook-spreadsheet-model", "r1-gemma-mlx-productivity-craftbook-spreadsheet-model", "r1-qwen-mlx-productivity-conflict-synthesis", "r1-qwen-llamacpp-productivity-conflict-synthesis", "r1-gemma-mlx-productivity-conflict-synthesis", "r1-gemma-llamacpp-productivity-conflict-synthesis", "r1-qwen-llamacpp-productivity-docblocks-theme-roundtrip", "r1-gemma-mlx-productivity-docblocks-theme-roundtrip", "r1-gemma-llamacpp-productivity-docblocks-theme-roundtrip", "r1-qwen-mlx-productivity-docblocks-theme-roundtrip", "r1-gemma-mlx-productivity-craftbook-powerpoint-deck", "r1-gemma-llamacpp-productivity-craftbook-powerpoint-deck", "r1-qwen-mlx-productivity-craftbook-powerpoint-deck", "r1-qwen-llamacpp-productivity-craftbook-powerpoint-deck", "r1-gemma-llamacpp-productivity-wikipedia-research-brief", "r1-qwen-mlx-productivity-wikipedia-research-brief", "r1-qwen-llamacpp-productivity-wikipedia-research-brief", "r1-gemma-mlx-productivity-wikipedia-research-brief", "r1-qwen-mlx-productivity-craftbook-week-plan", "r1-qwen-llamacpp-productivity-craftbook-week-plan", "r1-gemma-mlx-productivity-craftbook-week-plan", "r1-gemma-llamacpp-productivity-craftbook-week-plan", "r1-qwen-llamacpp-productivity-craftbook-ab-test-readout", "r1-gemma-mlx-productivity-craftbook-ab-test-readout", "r1-gemma-llamacpp-productivity-craftbook-ab-test-readout", "r1-qwen-mlx-productivity-craftbook-ab-test-readout", "r1-gemma-mlx-productivity-craftbook-annotated-bibliography", "r1-gemma-llamacpp-productivity-craftbook-annotated-bibliography", "r1-qwen-mlx-productivity-craftbook-annotated-bibliography", "r1-qwen-llamacpp-productivity-craftbook-annotated-bibliography", "r1-gemma-llamacpp-productivity-craftbook-research-to-document", "r1-qwen-mlx-productivity-craftbook-research-to-document", "r1-qwen-llamacpp-productivity-craftbook-research-to-document", "r1-gemma-mlx-productivity-craftbook-research-to-document", "r1-qwen-mlx-developer-fix-squisq-bugs", "r1-qwen-llamacpp-developer-fix-squisq-bugs", "r1-gemma-mlx-developer-fix-squisq-bugs", "r1-gemma-llamacpp-developer-fix-squisq-bugs", "r1-qwen-llamacpp-developer-dev-craftbook-routing", "r1-gemma-mlx-developer-dev-craftbook-routing", "r1-gemma-llamacpp-developer-dev-craftbook-routing", "r1-qwen-mlx-developer-dev-craftbook-routing", "r1-gemma-mlx-developer-interface-contract", "r1-gemma-llamacpp-developer-interface-contract", "r1-qwen-mlx-developer-interface-contract", "r1-qwen-llamacpp-developer-interface-contract", "r1-gemma-llamacpp-developer-craftbook-code-review", "r1-qwen-mlx-developer-craftbook-code-review", "r1-qwen-llamacpp-developer-craftbook-code-review", "r1-gemma-mlx-developer-craftbook-code-review", "r1-qwen-mlx-developer-craftbook-api-contract-review", "r1-qwen-llamacpp-developer-craftbook-api-contract-review", "r1-gemma-mlx-developer-craftbook-api-contract-review", "r1-gemma-llamacpp-developer-craftbook-api-contract-review", "r1-qwen-llamacpp-developer-craftbook-deep-security-review", "r1-gemma-mlx-developer-craftbook-deep-security-review", "r1-gemma-llamacpp-developer-craftbook-deep-security-review", "r1-qwen-mlx-developer-craftbook-deep-security-review", "r1-gemma-mlx-developer-large-pr-review", "r1-gemma-llamacpp-developer-large-pr-review", "r1-qwen-mlx-developer-large-pr-review", "r1-qwen-llamacpp-developer-large-pr-review", "r1-gemma-llamacpp-developer-craftbook-codemod-sweep", "r1-qwen-mlx-developer-craftbook-codemod-sweep", "r1-qwen-llamacpp-developer-craftbook-codemod-sweep", "r1-gemma-mlx-developer-craftbook-codemod-sweep", "r1-qwen-mlx-developer-craftbook-refactor-module", "r1-qwen-llamacpp-developer-craftbook-refactor-module", "r1-gemma-mlx-developer-craftbook-refactor-module", "r1-gemma-llamacpp-developer-craftbook-refactor-module", "r1-qwen-llamacpp-developer-craftbook-bug-fix-tdd", "r1-gemma-mlx-developer-craftbook-bug-fix-tdd", "r1-gemma-llamacpp-developer-craftbook-bug-fix-tdd", "r1-qwen-mlx-developer-craftbook-bug-fix-tdd", "r1-gemma-mlx-complex-work-craftbook-find-vs-create", "r1-gemma-llamacpp-complex-work-craftbook-find-vs-create", "r1-qwen-mlx-complex-work-craftbook-find-vs-create", "r1-qwen-llamacpp-complex-work-craftbook-find-vs-create", "r1-gemma-llamacpp-complex-work-craftbook-edit-midtask", "r1-qwen-mlx-complex-work-craftbook-edit-midtask", "r1-qwen-llamacpp-complex-work-craftbook-edit-midtask", "r1-gemma-mlx-complex-work-craftbook-edit-midtask", "r1-qwen-mlx-complex-work-craftbook-invoice-run", "r1-qwen-llamacpp-complex-work-craftbook-invoice-run", "r1-gemma-mlx-complex-work-craftbook-invoice-run", "r1-gemma-llamacpp-complex-work-craftbook-invoice-run", "r1-qwen-llamacpp-complex-work-craftbook-author-linear", "r1-gemma-mlx-complex-work-craftbook-author-linear", "r1-gemma-llamacpp-complex-work-craftbook-author-linear", "r1-qwen-mlx-complex-work-craftbook-author-linear", "r1-gemma-mlx-complex-work-craftbook-route-multi", "r1-gemma-llamacpp-complex-work-craftbook-route-multi", "r1-qwen-mlx-complex-work-craftbook-route-multi", "r1-qwen-llamacpp-complex-work-craftbook-route-multi", "r1-gemma-llamacpp-complex-work-craftbook-export-generalize", "r1-qwen-mlx-complex-work-craftbook-export-generalize", "r1-qwen-llamacpp-complex-work-craftbook-export-generalize", "r1-gemma-mlx-complex-work-craftbook-export-generalize", "r1-qwen-mlx-complex-work-craftbook-author-params", "r1-qwen-llamacpp-complex-work-craftbook-author-params", "r1-gemma-mlx-complex-work-craftbook-author-params", "r1-gemma-llamacpp-complex-work-craftbook-author-params", "r1-qwen-llamacpp-complex-work-craftbook-author-gate-script", "r1-gemma-mlx-complex-work-craftbook-author-gate-script", "r1-gemma-llamacpp-complex-work-craftbook-author-gate-script", "r1-qwen-mlx-complex-work-craftbook-author-gate-script", "r1-gemma-mlx-complex-work-craftbook-author-fanout", "r1-gemma-llamacpp-complex-work-craftbook-author-fanout", "r1-qwen-mlx-complex-work-craftbook-author-fanout", "r1-qwen-llamacpp-complex-work-craftbook-author-fanout", "r2-qwen-llamacpp-core-tictactoe", "r2-gemma-mlx-core-tictactoe", "r2-gemma-llamacpp-core-tictactoe", "r2-qwen-mlx-core-tictactoe", "r2-gemma-mlx-core-petshop", "r2-gemma-llamacpp-core-petshop", "r2-qwen-mlx-core-petshop", "r2-qwen-llamacpp-core-petshop", "r2-gemma-llamacpp-core-tankcombat", "r2-qwen-mlx-core-tankcombat", "r2-qwen-llamacpp-core-tankcombat", "r2-gemma-mlx-core-tankcombat", "r2-qwen-mlx-core-schema-migration", "r2-qwen-llamacpp-core-schema-migration", "r2-gemma-mlx-core-schema-migration", "r2-gemma-llamacpp-core-schema-migration", "r2-qwen-llamacpp-core-failing-tests-spec", "r2-gemma-mlx-core-failing-tests-spec", "r2-gemma-llamacpp-core-failing-tests-spec", "r2-qwen-mlx-core-failing-tests-spec", "r2-gemma-mlx-core-symptom-debug", "r2-gemma-llamacpp-core-symptom-debug", "r2-qwen-mlx-core-symptom-debug", "r2-qwen-llamacpp-core-symptom-debug", "r2-gemma-llamacpp-core-data-wrangle", "r2-qwen-mlx-core-data-wrangle", "r2-qwen-llamacpp-core-data-wrangle", "r2-gemma-mlx-core-data-wrangle", "r2-qwen-mlx-core-incident-postmortem", "r2-qwen-llamacpp-core-incident-postmortem", "r2-gemma-mlx-core-incident-postmortem", "r2-gemma-llamacpp-core-incident-postmortem", "r2-qwen-llamacpp-core-ops-runbook-anomaly", "r2-gemma-mlx-core-ops-runbook-anomaly", "r2-gemma-llamacpp-core-ops-runbook-anomaly", "r2-qwen-mlx-core-ops-runbook-anomaly", "r2-gemma-mlx-core-plan-and-estimate", "r2-gemma-llamacpp-core-plan-and-estimate", "r2-qwen-mlx-core-plan-and-estimate", "r2-qwen-llamacpp-core-plan-and-estimate", "r2-gemma-llamacpp-core-conflict-synthesis", "r2-qwen-mlx-core-conflict-synthesis", "r2-qwen-llamacpp-core-conflict-synthesis", "r2-gemma-mlx-core-conflict-synthesis", "r2-qwen-mlx-productivity-constrained-comms", "r2-qwen-llamacpp-productivity-constrained-comms", "r2-gemma-mlx-productivity-constrained-comms", "r2-gemma-llamacpp-productivity-constrained-comms", "r2-qwen-llamacpp-productivity-records-intake", "r2-gemma-mlx-productivity-records-intake", "r2-gemma-llamacpp-productivity-records-intake", "r2-qwen-mlx-productivity-records-intake", "r2-gemma-mlx-productivity-plan-and-estimate", "r2-gemma-llamacpp-productivity-plan-and-estimate", "r2-qwen-mlx-productivity-plan-and-estimate", "r2-qwen-llamacpp-productivity-plan-and-estimate", "r2-gemma-llamacpp-productivity-meeting-followup", "r2-qwen-mlx-productivity-meeting-followup", "r2-qwen-llamacpp-productivity-meeting-followup", "r2-gemma-mlx-productivity-meeting-followup", "r2-qwen-mlx-productivity-craftbook-spreadsheet-model", "r2-qwen-llamacpp-productivity-craftbook-spreadsheet-model", "r2-gemma-mlx-productivity-craftbook-spreadsheet-model", "r2-gemma-llamacpp-productivity-craftbook-spreadsheet-model", "r2-qwen-llamacpp-productivity-conflict-synthesis", "r2-gemma-mlx-productivity-conflict-synthesis", "r2-gemma-llamacpp-productivity-conflict-synthesis", "r2-qwen-mlx-productivity-conflict-synthesis", "r2-gemma-mlx-productivity-docblocks-theme-roundtrip", "r2-gemma-llamacpp-productivity-docblocks-theme-roundtrip", "r2-qwen-mlx-productivity-docblocks-theme-roundtrip", "r2-qwen-llamacpp-productivity-docblocks-theme-roundtrip", "r2-gemma-llamacpp-productivity-craftbook-powerpoint-deck", "r2-qwen-mlx-productivity-craftbook-powerpoint-deck", "r2-qwen-llamacpp-productivity-craftbook-powerpoint-deck", "r2-gemma-mlx-productivity-craftbook-powerpoint-deck", "r2-qwen-mlx-productivity-wikipedia-research-brief", "r2-qwen-llamacpp-productivity-wikipedia-research-brief", "r2-gemma-mlx-productivity-wikipedia-research-brief", "r2-gemma-llamacpp-productivity-wikipedia-research-brief", "r2-qwen-llamacpp-productivity-craftbook-week-plan", "r2-gemma-mlx-productivity-craftbook-week-plan", "r2-gemma-llamacpp-productivity-craftbook-week-plan", "r2-qwen-mlx-productivity-craftbook-week-plan", "r2-gemma-mlx-productivity-craftbook-ab-test-readout", "r2-gemma-llamacpp-productivity-craftbook-ab-test-readout", "r2-qwen-mlx-productivity-craftbook-ab-test-readout", "r2-qwen-llamacpp-productivity-craftbook-ab-test-readout", "r2-gemma-llamacpp-productivity-craftbook-annotated-bibliography", "r2-qwen-mlx-productivity-craftbook-annotated-bibliography", "r2-qwen-llamacpp-productivity-craftbook-annotated-bibliography", "r2-gemma-mlx-productivity-craftbook-annotated-bibliography", "r2-qwen-mlx-productivity-craftbook-research-to-document", "r2-qwen-llamacpp-productivity-craftbook-research-to-document", "r2-gemma-mlx-productivity-craftbook-research-to-document", "r2-gemma-llamacpp-productivity-craftbook-research-to-document", "r2-qwen-llamacpp-developer-fix-squisq-bugs", "r2-gemma-mlx-developer-fix-squisq-bugs", "r2-gemma-llamacpp-developer-fix-squisq-bugs", "r2-qwen-mlx-developer-fix-squisq-bugs", "r2-gemma-mlx-developer-dev-craftbook-routing", "r2-gemma-llamacpp-developer-dev-craftbook-routing", "r2-qwen-mlx-developer-dev-craftbook-routing", "r2-qwen-llamacpp-developer-dev-craftbook-routing", "r2-gemma-llamacpp-developer-interface-contract", "r2-qwen-mlx-developer-interface-contract", "r2-qwen-llamacpp-developer-interface-contract", "r2-gemma-mlx-developer-interface-contract", "r2-qwen-mlx-developer-craftbook-code-review", "r2-qwen-llamacpp-developer-craftbook-code-review", "r2-gemma-mlx-developer-craftbook-code-review", "r2-gemma-llamacpp-developer-craftbook-code-review", "r2-qwen-llamacpp-developer-craftbook-api-contract-review", "r2-gemma-mlx-developer-craftbook-api-contract-review", "r2-gemma-llamacpp-developer-craftbook-api-contract-review", "r2-qwen-mlx-developer-craftbook-api-contract-review", "r2-gemma-mlx-developer-craftbook-deep-security-review", "r2-gemma-llamacpp-developer-craftbook-deep-security-review", "r2-qwen-mlx-developer-craftbook-deep-security-review", "r2-qwen-llamacpp-developer-craftbook-deep-security-review", "r2-gemma-llamacpp-developer-large-pr-review", "r2-qwen-mlx-developer-large-pr-review", "r2-qwen-llamacpp-developer-large-pr-review", "r2-gemma-mlx-developer-large-pr-review", "r2-qwen-mlx-developer-craftbook-codemod-sweep", "r2-qwen-llamacpp-developer-craftbook-codemod-sweep", "r2-gemma-mlx-developer-craftbook-codemod-sweep", "r2-gemma-llamacpp-developer-craftbook-codemod-sweep", "r2-qwen-llamacpp-developer-craftbook-refactor-module", "r2-gemma-mlx-developer-craftbook-refactor-module", "r2-gemma-llamacpp-developer-craftbook-refactor-module", "r2-qwen-mlx-developer-craftbook-refactor-module", "r2-gemma-mlx-developer-craftbook-bug-fix-tdd", "r2-gemma-llamacpp-developer-craftbook-bug-fix-tdd", "r2-qwen-mlx-developer-craftbook-bug-fix-tdd", "r2-qwen-llamacpp-developer-craftbook-bug-fix-tdd", "r2-gemma-llamacpp-complex-work-craftbook-find-vs-create", "r2-qwen-mlx-complex-work-craftbook-find-vs-create", "r2-qwen-llamacpp-complex-work-craftbook-find-vs-create", "r2-gemma-mlx-complex-work-craftbook-find-vs-create", "r2-qwen-mlx-complex-work-craftbook-edit-midtask", "r2-qwen-llamacpp-complex-work-craftbook-edit-midtask", "r2-gemma-mlx-complex-work-craftbook-edit-midtask", "r2-gemma-llamacpp-complex-work-craftbook-edit-midtask", "r2-qwen-llamacpp-complex-work-craftbook-invoice-run", "r2-gemma-mlx-complex-work-craftbook-invoice-run", "r2-gemma-llamacpp-complex-work-craftbook-invoice-run", "r2-qwen-mlx-complex-work-craftbook-invoice-run", "r2-gemma-mlx-complex-work-craftbook-author-linear", "r2-gemma-llamacpp-complex-work-craftbook-author-linear", "r2-qwen-mlx-complex-work-craftbook-author-linear", "r2-qwen-llamacpp-complex-work-craftbook-author-linear", "r2-gemma-llamacpp-complex-work-craftbook-route-multi", "r2-qwen-mlx-complex-work-craftbook-route-multi", "r2-qwen-llamacpp-complex-work-craftbook-route-multi", "r2-gemma-mlx-complex-work-craftbook-route-multi", "r2-qwen-mlx-complex-work-craftbook-export-generalize", "r2-qwen-llamacpp-complex-work-craftbook-export-generalize", "r2-gemma-mlx-complex-work-craftbook-export-generalize", "r2-gemma-llamacpp-complex-work-craftbook-export-generalize", "r2-qwen-llamacpp-complex-work-craftbook-author-params", "r2-gemma-mlx-complex-work-craftbook-author-params", "r2-gemma-llamacpp-complex-work-craftbook-author-params", "r2-qwen-mlx-complex-work-craftbook-author-params", "r2-gemma-mlx-complex-work-craftbook-author-gate-script", "r2-gemma-llamacpp-complex-work-craftbook-author-gate-script", "r2-qwen-mlx-complex-work-craftbook-author-gate-script", "r2-qwen-llamacpp-complex-work-craftbook-author-gate-script", "r2-gemma-llamacpp-complex-work-craftbook-author-fanout", "r2-qwen-mlx-complex-work-craftbook-author-fanout", "r2-qwen-llamacpp-complex-work-craftbook-author-fanout", "r2-gemma-mlx-complex-work-craftbook-author-fanout", "r3-gemma-mlx-core-tictactoe", "r3-gemma-llamacpp-core-tictactoe", "r3-qwen-mlx-core-tictactoe", "r3-qwen-llamacpp-core-tictactoe", "r3-gemma-llamacpp-core-petshop", "r3-qwen-mlx-core-petshop", "r3-qwen-llamacpp-core-petshop", "r3-gemma-mlx-core-petshop", "r3-qwen-mlx-core-tankcombat", "r3-qwen-llamacpp-core-tankcombat", "r3-gemma-mlx-core-tankcombat", "r3-gemma-llamacpp-core-tankcombat", "r3-qwen-llamacpp-core-schema-migration", "r3-gemma-mlx-core-schema-migration", "r3-gemma-llamacpp-core-schema-migration", "r3-qwen-mlx-core-schema-migration", "r3-gemma-mlx-core-failing-tests-spec", "r3-gemma-llamacpp-core-failing-tests-spec", "r3-qwen-mlx-core-failing-tests-spec", "r3-qwen-llamacpp-core-failing-tests-spec", "r3-gemma-llamacpp-core-symptom-debug", "r3-qwen-mlx-core-symptom-debug", "r3-qwen-llamacpp-core-symptom-debug", "r3-gemma-mlx-core-symptom-debug", "r3-qwen-mlx-core-data-wrangle", "r3-qwen-llamacpp-core-data-wrangle", "r3-gemma-mlx-core-data-wrangle", "r3-gemma-llamacpp-core-data-wrangle", "r3-qwen-llamacpp-core-incident-postmortem", "r3-gemma-mlx-core-incident-postmortem", "r3-gemma-llamacpp-core-incident-postmortem", "r3-qwen-mlx-core-incident-postmortem", "r3-gemma-mlx-core-ops-runbook-anomaly", "r3-gemma-llamacpp-core-ops-runbook-anomaly", "r3-qwen-mlx-core-ops-runbook-anomaly", "r3-qwen-llamacpp-core-ops-runbook-anomaly", "r3-gemma-llamacpp-core-plan-and-estimate", "r3-qwen-mlx-core-plan-and-estimate", "r3-qwen-llamacpp-core-plan-and-estimate", "r3-gemma-mlx-core-plan-and-estimate", "r3-qwen-mlx-core-conflict-synthesis", "r3-qwen-llamacpp-core-conflict-synthesis", "r3-gemma-mlx-core-conflict-synthesis", "r3-gemma-llamacpp-core-conflict-synthesis", "r3-qwen-llamacpp-productivity-constrained-comms", "r3-gemma-mlx-productivity-constrained-comms", "r3-gemma-llamacpp-productivity-constrained-comms", "r3-qwen-mlx-productivity-constrained-comms", "r3-gemma-mlx-productivity-records-intake", "r3-gemma-llamacpp-productivity-records-intake", "r3-qwen-mlx-productivity-records-intake", "r3-qwen-llamacpp-productivity-records-intake", "r3-gemma-llamacpp-productivity-plan-and-estimate", "r3-qwen-mlx-productivity-plan-and-estimate", "r3-qwen-llamacpp-productivity-plan-and-estimate", "r3-gemma-mlx-productivity-plan-and-estimate", "r3-qwen-mlx-productivity-meeting-followup", "r3-qwen-llamacpp-productivity-meeting-followup", "r3-gemma-mlx-productivity-meeting-followup", "r3-gemma-llamacpp-productivity-meeting-followup", "r3-qwen-llamacpp-productivity-craftbook-spreadsheet-model", "r3-gemma-mlx-productivity-craftbook-spreadsheet-model", "r3-gemma-llamacpp-productivity-craftbook-spreadsheet-model", "r3-qwen-mlx-productivity-craftbook-spreadsheet-model", "r3-gemma-mlx-productivity-conflict-synthesis", "r3-gemma-llamacpp-productivity-conflict-synthesis", "r3-qwen-mlx-productivity-conflict-synthesis", "r3-qwen-llamacpp-productivity-conflict-synthesis", "r3-gemma-llamacpp-productivity-docblocks-theme-roundtrip", "r3-qwen-mlx-productivity-docblocks-theme-roundtrip", "r3-qwen-llamacpp-productivity-docblocks-theme-roundtrip", "r3-gemma-mlx-productivity-docblocks-theme-roundtrip", "r3-qwen-mlx-productivity-craftbook-powerpoint-deck", "r3-qwen-llamacpp-productivity-craftbook-powerpoint-deck", "r3-gemma-mlx-productivity-craftbook-powerpoint-deck", "r3-gemma-llamacpp-productivity-craftbook-powerpoint-deck", "r3-qwen-llamacpp-productivity-wikipedia-research-brief", "r3-gemma-mlx-productivity-wikipedia-research-brief", "r3-gemma-llamacpp-productivity-wikipedia-research-brief", "r3-qwen-mlx-productivity-wikipedia-research-brief", "r3-gemma-mlx-productivity-craftbook-week-plan", "r3-gemma-llamacpp-productivity-craftbook-week-plan", "r3-qwen-mlx-productivity-craftbook-week-plan", "r3-qwen-llamacpp-productivity-craftbook-week-plan", "r3-gemma-llamacpp-productivity-craftbook-ab-test-readout", "r3-qwen-mlx-productivity-craftbook-ab-test-readout", "r3-qwen-llamacpp-productivity-craftbook-ab-test-readout", "r3-gemma-mlx-productivity-craftbook-ab-test-readout", "r3-qwen-mlx-productivity-craftbook-annotated-bibliography", "r3-qwen-llamacpp-productivity-craftbook-annotated-bibliography", "r3-gemma-mlx-productivity-craftbook-annotated-bibliography", "r3-gemma-llamacpp-productivity-craftbook-annotated-bibliography", "r3-qwen-llamacpp-productivity-craftbook-research-to-document", "r3-gemma-mlx-productivity-craftbook-research-to-document", "r3-gemma-llamacpp-productivity-craftbook-research-to-document", "r3-qwen-mlx-productivity-craftbook-research-to-document", "r3-gemma-mlx-developer-fix-squisq-bugs", "r3-gemma-llamacpp-developer-fix-squisq-bugs", "r3-qwen-mlx-developer-fix-squisq-bugs", "r3-qwen-llamacpp-developer-fix-squisq-bugs", "r3-gemma-llamacpp-developer-dev-craftbook-routing", "r3-qwen-mlx-developer-dev-craftbook-routing", "r3-qwen-llamacpp-developer-dev-craftbook-routing", "r3-gemma-mlx-developer-dev-craftbook-routing", "r3-qwen-mlx-developer-interface-contract", "r3-qwen-llamacpp-developer-interface-contract", "r3-gemma-mlx-developer-interface-contract", "r3-gemma-llamacpp-developer-interface-contract", "r3-qwen-llamacpp-developer-craftbook-code-review", "r3-gemma-mlx-developer-craftbook-code-review", "r3-gemma-llamacpp-developer-craftbook-code-review", "r3-qwen-mlx-developer-craftbook-code-review", "r3-gemma-mlx-developer-craftbook-api-contract-review", "r3-gemma-llamacpp-developer-craftbook-api-contract-review", "r3-qwen-mlx-developer-craftbook-api-contract-review", "r3-qwen-llamacpp-developer-craftbook-api-contract-review", "r3-gemma-llamacpp-developer-craftbook-deep-security-review", "r3-qwen-mlx-developer-craftbook-deep-security-review", "r3-qwen-llamacpp-developer-craftbook-deep-security-review", "r3-gemma-mlx-developer-craftbook-deep-security-review", "r3-qwen-mlx-developer-large-pr-review", "r3-qwen-llamacpp-developer-large-pr-review", "r3-gemma-mlx-developer-large-pr-review", "r3-gemma-llamacpp-developer-large-pr-review", "r3-qwen-llamacpp-developer-craftbook-codemod-sweep", "r3-gemma-mlx-developer-craftbook-codemod-sweep", "r3-gemma-llamacpp-developer-craftbook-codemod-sweep", "r3-qwen-mlx-developer-craftbook-codemod-sweep", "r3-gemma-mlx-developer-craftbook-refactor-module", "r3-gemma-llamacpp-developer-craftbook-refactor-module", "r3-qwen-mlx-developer-craftbook-refactor-module", "r3-qwen-llamacpp-developer-craftbook-refactor-module", "r3-gemma-llamacpp-developer-craftbook-bug-fix-tdd", "r3-qwen-mlx-developer-craftbook-bug-fix-tdd", "r3-qwen-llamacpp-developer-craftbook-bug-fix-tdd", "r3-gemma-mlx-developer-craftbook-bug-fix-tdd", "r3-qwen-mlx-complex-work-craftbook-find-vs-create", "r3-qwen-llamacpp-complex-work-craftbook-find-vs-create", "r3-gemma-mlx-complex-work-craftbook-find-vs-create", "r3-gemma-llamacpp-complex-work-craftbook-find-vs-create", "r3-qwen-llamacpp-complex-work-craftbook-edit-midtask", "r3-gemma-mlx-complex-work-craftbook-edit-midtask", "r3-gemma-llamacpp-complex-work-craftbook-edit-midtask", "r3-qwen-mlx-complex-work-craftbook-edit-midtask", "r3-gemma-mlx-complex-work-craftbook-invoice-run", "r3-gemma-llamacpp-complex-work-craftbook-invoice-run", "r3-qwen-mlx-complex-work-craftbook-invoice-run", "r3-qwen-llamacpp-complex-work-craftbook-invoice-run", "r3-gemma-llamacpp-complex-work-craftbook-author-linear", "r3-qwen-mlx-complex-work-craftbook-author-linear", "r3-qwen-llamacpp-complex-work-craftbook-author-linear", "r3-gemma-mlx-complex-work-craftbook-author-linear", "r3-qwen-mlx-complex-work-craftbook-route-multi", "r3-qwen-llamacpp-complex-work-craftbook-route-multi", "r3-gemma-mlx-complex-work-craftbook-route-multi", "r3-gemma-llamacpp-complex-work-craftbook-route-multi", "r3-qwen-llamacpp-complex-work-craftbook-export-generalize", "r3-gemma-mlx-complex-work-craftbook-export-generalize", "r3-gemma-llamacpp-complex-work-craftbook-export-generalize", "r3-qwen-mlx-complex-work-craftbook-export-generalize", "r3-gemma-mlx-complex-work-craftbook-author-params", "r3-gemma-llamacpp-complex-work-craftbook-author-params", "r3-qwen-mlx-complex-work-craftbook-author-params", "r3-qwen-llamacpp-complex-work-craftbook-author-params", "r3-gemma-llamacpp-complex-work-craftbook-author-gate-script", "r3-qwen-mlx-complex-work-craftbook-author-gate-script", "r3-qwen-llamacpp-complex-work-craftbook-author-gate-script", "r3-gemma-mlx-complex-work-craftbook-author-gate-script", "r3-qwen-mlx-complex-work-craftbook-author-fanout", "r3-qwen-llamacpp-complex-work-craftbook-author-fanout", "r3-gemma-mlx-complex-work-craftbook-author-fanout", "r3-gemma-llamacpp-complex-work-craftbook-author-fanout" ] }, "observedMlxPrefill": { "available": 258, "totalMlxTrials": 258, "observations": 2175 } }, "coverageNote": "prefillThroughput retains aggregate-rate availability. Recovered MLX observations are additional, partial evidence, tracked separately under observedMlxPrefill.", "trials": [ { "cell": { "id": "r1-qwen-mlx-core-tictactoe", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "tictactoe", "repetition": 1 }, "trialId": "tictactoe-mlx-qwen3-8-27b-q4-2026-10-28T00-10-08-465Z-d99v", "success": true, "includedInCapability": true, "durationMs": 127947, "timing": { "startedAt": "2026-10-28T00:10:08.465Z", "finishedAt": "2026-10-28T00:12:16.412Z", "timeToFirstArtifactMs": 117940, "timeToLastArtifactWriteMs": 117940, "timeToFirstTokenMs": 22867, "firstTurnTtftMs": 20967, "timeToFirstToolCallMs": 24436 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 19125, "peakCpuPercent": 115.7, "firstAtMs": 0, "lastAtMs": 120513 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 90170, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16636, "totalOutputTokens": 2937, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11977, "todayTokensCached": 11854, "todayTokensOut": 382, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11977, "totalTokensCached": 11854, "totalTokensOut": 382, "totalCost": 0, "medianOutputTokensPerSec": 37.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.3, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11977, "cachedInputTokens": 11854, "outputTokens": 382, "cost": 0, "durationMs": 48480, "outputTokensPerSec": 37.32650004620359, "at": "2026-10-28T00:10:58.845Z" }, "lastUpdated": "2026-10-28T00:10:58.845Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:12:12.062Z" } }, "source": "engine-log" }, "derived": { "durationMs": 122342, "meanTokensPerSec": 43, "genTokensPerSec": 43, "genTokens": 2937, "promptTokensPerSec": null, "peakRssMb": 19125, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11977, "output": 382 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 692.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-tictactoe", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "tictactoe", "repetition": 1 }, "trialId": "tictactoe-qwen3-8-27b-q4-2026-10-28T00-13-20-787Z-woaz", "success": true, "includedInCapability": true, "durationMs": 127605, "timing": { "startedAt": "2026-10-28T00:13:20.788Z", "finishedAt": "2026-10-28T00:15:28.393Z", "timeToFirstArtifactMs": 118318, "timeToLastArtifactWriteMs": 118322, "timeToFirstTokenMs": 23005, "firstTurnTtftMs": 21235, "timeToFirstToolCallMs": 35796 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 40705, "peakCpuPercent": 131.2, "firstAtMs": 0, "lastAtMs": 120486 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 101420, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 11288, "totalOutputTokens": 2848, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22870, "todayTokensCached": 21405, "todayTokensOut": 383, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22870, "totalTokensCached": 21405, "totalTokensOut": 383, "totalCost": 0, "medianOutputTokensPerSec": 32, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8195, "cachedInputTokens": 7561, "outputTokens": 92, "cost": 0, "durationMs": 31273, "outputTokensPerSec": 29.321771318538833, "at": "2026-10-28T00:14:39.077Z" }, "lastUpdated": "2026-10-28T00:14:39.077Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:15:24.096Z" } }, "source": "engine-log" }, "derived": { "durationMs": 122272, "meanTokensPerSec": 35.32, "genTokensPerSec": 35.32, "genTokens": 2848, "promptTokensPerSec": 451.12, "peakRssMb": 40705, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22870, "output": 383 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-tictactoe", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "tictactoe", "repetition": 1 }, "trialId": "tictactoe-mlx-gemma4-12b-q4-2026-10-28T00-16-25-025Z-m3bu", "success": true, "includedInCapability": true, "durationMs": 116933, "timing": { "startedAt": "2026-10-28T00:16:25.025Z", "finishedAt": "2026-10-28T00:18:21.958Z", "timeToFirstArtifactMs": 55110, "timeToLastArtifactWriteMs": 113358, "timeToFirstTokenMs": 14169, "firstTurnTtftMs": 12271, "timeToFirstToolCallMs": 39047 }, "metrics": { "process": { "sampleCount": 23, "peakRssMb": 10873, "peakCpuPercent": 102.5, "firstAtMs": 0, "lastAtMs": 110581 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 73128, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 28387, "totalOutputTokens": 2405, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 28339, "todayTokensCached": 20095, "todayTokensOut": 212, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 28339, "totalTokensCached": 20095, "totalTokensOut": 212, "totalCost": 0, "medianOutputTokensPerSec": 42.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.3, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11284, "cachedInputTokens": 9782, "outputTokens": 31, "cost": 0, "durationMs": 57593, "outputTokensPerSec": 42.315551666614176, "at": "2026-10-28T00:18:18.383Z" }, "lastUpdated": "2026-10-28T00:18:18.383Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:18:18.661Z" } }, "source": "engine-log" }, "derived": { "durationMs": 112378, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 2405, "promptTokensPerSec": null, "peakRssMb": 10873, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 28339, "output": 212 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1447, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-tictactoe", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "tictactoe", "repetition": 1 }, "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T00-19-19-206Z-9ffz", "success": true, "includedInCapability": true, "durationMs": 50592, "timing": { "startedAt": "2026-10-28T00:19:19.206Z", "finishedAt": "2026-10-28T00:20:09.798Z", "timeToFirstArtifactMs": 42446, "timeToLastArtifactWriteMs": 42450, "timeToFirstTokenMs": 11725, "firstTurnTtftMs": 9895, "timeToFirstToolCallMs": 18090 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 18124, "peakCpuPercent": 106.4, "firstAtMs": 0, "lastAtMs": 45190 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 78274, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13567, "totalOutputTokens": 1959, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6949, "todayTokensCached": 6777, "todayTokensOut": 18, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6949, "totalTokensCached": 6777, "totalTokensOut": 18, "totalCost": 0, "medianOutputTokensPerSec": 57.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 57.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6949, "cachedInputTokens": 6777, "outputTokens": 18, "cost": 0, "durationMs": 17843, "outputTokensPerSec": 57.92933302892037, "at": "2026-10-28T00:19:38.880Z" }, "lastUpdated": "2026-10-28T00:19:38.880Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:20:07.221Z" } }, "source": "engine-log" }, "derived": { "durationMs": 46974, "meanTokensPerSec": 99.88, "genTokensPerSec": 99.88, "genTokens": 1959, "promptTokensPerSec": 1149.05, "peakRssMb": 18124, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6949, "output": 18 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-core-petshop", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "petshop", "repetition": 1 }, "trialId": "petshop-qwen3-8-27b-q4-2026-10-28T00-21-36-729Z-0s2q", "success": true, "includedInCapability": true, "durationMs": 515612, "timing": { "startedAt": "2026-10-28T00:21:36.730Z", "finishedAt": "2026-10-28T00:30:12.342Z", "timeToFirstArtifactMs": 387773, "timeToLastArtifactWriteMs": 387773, "timeToFirstTokenMs": 19666, "firstTurnTtftMs": 17832, "timeToFirstToolCallMs": 47267 }, "metrics": { "process": { "sampleCount": 101, "peakRssMb": 55762, "peakCpuPercent": 136.3, "firstAtMs": 0, "lastAtMs": 502049 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 101, "peakUsedMb": 119581, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 53368, "totalOutputTokens": 8380, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 59436, "todayTokensCached": 55478, "todayTokensOut": 1183, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 59436, "totalTokensCached": 55478, "totalTokensOut": 1183, "totalCost": 0, "medianOutputTokensPerSec": 29.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.7, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8759, "cachedInputTokens": 7870, "outputTokens": 156, "cost": 0, "durationMs": 21948, "outputTokensPerSec": 24.561396836745626, "at": "2026-10-28T00:28:26.456Z" }, "lastUpdated": "2026-10-28T00:28:26.456Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:30:00.855Z" } }, "source": "engine-log" }, "derived": { "durationMs": 503076, "meanTokensPerSec": 28.32, "genTokensPerSec": 28.32, "genTokens": 8380, "promptTokensPerSec": 427.21, "peakRssMb": 55762, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 59436, "output": 1183 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-petshop", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "petshop", "repetition": 1 }, "trialId": "petshop-mlx-gemma4-12b-q4-2026-10-28T00-30-51-467Z-02o4", "success": false, "includedInCapability": true, "durationMs": 1126083, "timing": { "startedAt": "2026-10-28T00:30:51.468Z", "finishedAt": "2026-10-28T00:49:37.551Z", "timeToFirstArtifactMs": 76063, "timeToLastArtifactWriteMs": 1116924, "timeToFirstTokenMs": 16774, "firstTurnTtftMs": 15092, "timeToFirstToolCallMs": 32596 }, "metrics": { "process": { "sampleCount": 223, "peakRssMb": 12860, "peakCpuPercent": 128.1, "firstAtMs": 0, "lastAtMs": 1115920 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 223, "peakUsedMb": 74662, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 106052, "totalOutputTokens": 45047, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 182556, "todayTokensCached": 159044, "todayTokensOut": 22724, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 182556, "totalTokensCached": 159044, "totalTokensOut": 22724, "totalCost": 0, "medianOutputTokensPerSec": 34.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.4, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6371, "cachedInputTokens": 0, "outputTokens": 1221, "cost": 0, "durationMs": 52602, "outputTokensPerSec": 34.356109391415195, "at": "2026-10-28T00:49:28.392Z" }, "lastUpdated": "2026-10-28T00:49:28.392Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:49:28.986Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1116450, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 45047, "promptTokensPerSec": null, "peakRssMb": 12860, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 182556, "output": 22724 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 853, "sampleCount": 18, "structuredSampleCount": 0, "firstTokenSampleCount": 18, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-petshop", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "petshop", "repetition": 1 }, "trialId": "petshop-gemma4-12b-q4-2026-10-28T00-50-00-098Z-5ams", "success": true, "includedInCapability": true, "durationMs": 547714, "timing": { "startedAt": "2026-10-28T00:50:00.099Z", "finishedAt": "2026-10-28T00:59:07.813Z", "timeToFirstArtifactMs": 82284, "timeToLastArtifactWriteMs": 290142, "timeToFirstTokenMs": 12033, "firstTurnTtftMs": 10133, "timeToFirstToolCallMs": 20866 }, "metrics": { "process": { "sampleCount": 109, "peakRssMb": 31552, "peakCpuPercent": 118.3, "firstAtMs": 0, "lastAtMs": 542140 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 109, "peakUsedMb": 96862, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 160905, "totalOutputTokens": 18802, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 127670, "todayTokensCached": 103678, "todayTokensOut": 2993, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 127670, "totalTokensCached": 103678, "totalTokensOut": 2993, "totalCost": 0, "medianOutputTokensPerSec": 58.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 58.5, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18165, "cachedInputTokens": 17706, "outputTokens": 35, "cost": 0, "durationMs": 49401, "outputTokensPerSec": 54.52842534737814, "at": "2026-10-28T00:58:59.995Z" }, "lastUpdated": "2026-10-28T00:58:59.995Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T00:59:04.575Z" } }, "source": "engine-log" }, "derived": { "durationMs": 543419, "meanTokensPerSec": 68.28, "genTokensPerSec": 68.28, "genTokens": 18802, "promptTokensPerSec": 934.73, "peakRssMb": 31552, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 127670, "output": 2993 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-petshop", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "petshop", "repetition": 1 }, "trialId": "petshop-mlx-qwen3-8-27b-q4-2026-10-28T00-59-53-840Z-tkdx", "success": false, "includedInCapability": true, "durationMs": 1656717, "timing": { "startedAt": "2026-10-28T00:59:53.841Z", "finishedAt": "2026-10-28T01:27:30.558Z", "timeToFirstArtifactMs": 558359, "timeToLastArtifactWriteMs": 558359, "timeToFirstTokenMs": 27909, "firstTurnTtftMs": 26232, "timeToFirstToolCallMs": 29900 }, "metrics": { "process": { "sampleCount": 329, "peakRssMb": 19486, "peakCpuPercent": 117.5, "firstAtMs": 0, "lastAtMs": 1646592 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 329, "peakUsedMb": 95071, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 65939, "totalOutputTokens": 39041, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 58786, "todayTokensCached": 46219, "todayTokensOut": 1950, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 58786, "totalTokensCached": 46219, "totalTokensOut": 1950, "totalCost": 0, "medianOutputTokensPerSec": 24.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.9, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12035, "cachedInputTokens": 0, "outputTokens": 878, "cost": 0, "durationMs": 80507, "outputTokensPerSec": 15.752221660257723, "at": "2026-10-28T01:10:38.460Z" }, "lastUpdated": "2026-10-28T01:10:38.460Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:27:22.036Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1647120, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 39041, "promptTokensPerSec": null, "peakRssMb": 19486, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 58786, "output": 1950 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 561, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-core-tankcombat", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "tankcombat", "repetition": 1 }, "trialId": "tankcombat-mlx-gemma4-12b-q4-2026-10-28T01-27-56-400Z-0tfe", "success": true, "includedInCapability": true, "durationMs": 106461, "timing": { "startedAt": "2026-10-28T01:27:56.401Z", "finishedAt": "2026-10-28T01:29:42.862Z", "timeToFirstArtifactMs": 96547, "timeToLastArtifactWriteMs": 96551, "timeToFirstTokenMs": 17063, "firstTurnTtftMs": 15389, "timeToFirstToolCallMs": 39991 }, "metrics": { "process": { "sampleCount": 21, "peakRssMb": 11222, "peakCpuPercent": 120.3, "firstAtMs": 0, "lastAtMs": 100438 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 73745, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16800, "totalOutputTokens": 2554, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 16768, "todayTokensCached": 10464, "todayTokensOut": 1845, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 16768, "totalTokensCached": 10464, "totalTokensOut": 1845, "totalCost": 0, "medianOutputTokensPerSec": 41.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 41.1, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6119, "cachedInputTokens": 0, "outputTokens": 1820, "cost": 0, "durationMs": 50000, "outputTokensPerSec": 39.449589203311085, "at": "2026-10-28T01:29:32.952Z" }, "lastUpdated": "2026-10-28T01:29:32.952Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:29:40.207Z" } }, "source": "engine-log" }, "derived": { "durationMs": 102743, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 2554, "promptTokensPerSec": null, "peakRssMb": 11222, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 16768, "output": 1845 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1378, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-tankcombat", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "tankcombat", "repetition": 1 }, "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T01-30-31-823Z-se4c", "success": true, "includedInCapability": true, "durationMs": 53662, "timing": { "startedAt": "2026-10-28T01:30:31.824Z", "finishedAt": "2026-10-28T01:31:25.486Z", "timeToFirstArtifactMs": 45849, "timeToLastArtifactWriteMs": 45853, "timeToFirstTokenMs": 9940, "firstTurnTtftMs": 8034, "timeToFirstToolCallMs": 15064 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 18063, "peakCpuPercent": 109.3, "firstAtMs": 0, "lastAtMs": 45180 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 77804, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13278, "totalOutputTokens": 2607, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7114, "todayTokensCached": 6942, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7114, "totalTokensCached": 6942, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 73.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 73.8, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7114, "cachedInputTokens": 6942, "outputTokens": 23, "cost": 0, "durationMs": 14348, "outputTokensPerSec": 73.80544214492033, "at": "2026-10-28T01:30:48.078Z" }, "lastUpdated": "2026-10-28T01:30:48.078Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:31:20.360Z" } }, "source": "engine-log" }, "derived": { "durationMs": 47486, "meanTokensPerSec": 117.61, "genTokensPerSec": 117.61, "genTokens": 2607, "promptTokensPerSec": 1517.85, "peakRssMb": 18063, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7114, "output": 23 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-tankcombat", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "tankcombat", "repetition": 1 }, "trialId": "tankcombat-mlx-qwen3-8-27b-q4-2026-10-28T01-32-10-142Z-baus", "success": true, "includedInCapability": true, "durationMs": 383161, "timing": { "startedAt": "2026-10-28T01:32:10.143Z", "finishedAt": "2026-10-28T01:38:33.304Z", "timeToFirstArtifactMs": 371236, "timeToLastArtifactWriteMs": 371236, "timeToFirstTokenMs": 22984, "firstTurnTtftMs": 21309, "timeToFirstToolCallMs": 34689 }, "metrics": { "process": { "sampleCount": 75, "peakRssMb": 19061, "peakCpuPercent": 150.2, "firstAtMs": 0, "lastAtMs": 371545 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 75, "peakUsedMb": 90897, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15060, "totalOutputTokens": 10596, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12128, "todayTokensCached": 12013, "todayTokensOut": 668, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12128, "totalTokensCached": 12013, "totalTokensOut": 668, "totalCost": 0, "medianOutputTokensPerSec": 39.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 39.2, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12128, "cachedInputTokens": 12013, "outputTokens": 668, "cost": 0, "durationMs": 61152, "outputTokensPerSec": 39.21272652250562, "at": "2026-10-28T01:33:12.970Z" }, "lastUpdated": "2026-10-28T01:33:12.970Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:38:25.293Z" } }, "source": "engine-log" }, "derived": { "durationMs": 374084, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 10596, "promptTokensPerSec": null, "peakRssMb": 19061, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12128, "output": 668 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 769.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-tankcombat", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "tankcombat", "repetition": 1 }, "trialId": "tankcombat-qwen3-8-27b-q4-2026-10-28T01-39-15-487Z-384l", "success": true, "includedInCapability": true, "durationMs": 202029, "timing": { "startedAt": "2026-10-28T01:39:15.487Z", "finishedAt": "2026-10-28T01:42:37.516Z", "timeToFirstArtifactMs": 189478, "timeToLastArtifactWriteMs": 189483, "timeToFirstTokenMs": 24707, "firstTurnTtftMs": 22832, "timeToFirstToolCallMs": 39787 }, "metrics": { "process": { "sampleCount": 39, "peakRssMb": 41470, "peakCpuPercent": 192.8, "firstAtMs": 0, "lastAtMs": 190749 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 39, "peakUsedMb": 101380, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12698, "totalOutputTokens": 5240, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 24272, "todayTokensCached": 22672, "todayTokensOut": 590, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 24272, "totalTokensCached": 22672, "totalTokensOut": 590, "totalCost": 0, "medianOutputTokensPerSec": 29.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.3, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8893, "cachedInputTokens": 8382, "outputTokens": 87, "cost": 0, "durationMs": 37634, "outputTokensPerSec": 27.07205068895221, "at": "2026-10-28T01:40:53.437Z" }, "lastUpdated": "2026-10-28T01:40:53.437Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:42:29.577Z" } }, "source": "engine-log" }, "derived": { "durationMs": 193039, "meanTokensPerSec": 35.65, "genTokensPerSec": 35.65, "genTokens": 5240, "promptTokensPerSec": 430.25, "peakRssMb": 41470, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 24272, "output": 590 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-core-schema-migration", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "schema-migration", "repetition": 1 }, "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T01-43-19-170Z-l6x5", "success": true, "includedInCapability": true, "durationMs": 196128, "timing": { "startedAt": "2026-10-28T01:43:19.171Z", "finishedAt": "2026-10-28T01:46:35.299Z", "timeToFirstArtifactMs": 25663, "timeToLastArtifactWriteMs": 192025, "timeToFirstTokenMs": 10353, "firstTurnTtftMs": 8546, "timeToFirstToolCallMs": 14723 }, "metrics": { "process": { "sampleCount": 39, "peakRssMb": 24187, "peakCpuPercent": 119, "firstAtMs": 0, "lastAtMs": 190791 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 39, "peakUsedMb": 78212, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 89520, "totalOutputTokens": 5958, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 62699, "todayTokensCached": 25377, "todayTokensOut": 2195, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 62699, "totalTokensCached": 25377, "totalTokensOut": 2195, "totalCost": 0, "medianOutputTokensPerSec": 76.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 76.7, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12052, "outputTokens": 746, "cost": 0, "durationMs": 24831, "outputTokensPerSec": 58.9218705487295, "at": "2026-10-28T01:46:31.196Z" }, "lastUpdated": "2026-10-28T01:46:31.196Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:46:32.043Z" } }, "source": "engine-log" }, "derived": { "durationMs": 191825, "meanTokensPerSec": 73.64, "genTokensPerSec": 73.64, "genTokens": 5958, "promptTokensPerSec": 1031.58, "peakRssMb": 24187, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 62699, "output": 2195 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-schema-migration", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "schema-migration", "repetition": 1 }, "trialId": "schema-migration-mlx-qwen3-8-27b-q4-2026-10-28T01-46-59-305Z-hzvp", "success": true, "includedInCapability": true, "durationMs": 219257, "timing": { "startedAt": "2026-10-28T01:46:59.306Z", "finishedAt": "2026-10-28T01:50:38.563Z", "timeToFirstArtifactMs": 32765, "timeToLastArtifactWriteMs": 216258, "timeToFirstTokenMs": 16052, "firstTurnTtftMs": 14398, "timeToFirstToolCallMs": 20031 }, "metrics": { "process": { "sampleCount": 44, "peakRssMb": 18250, "peakCpuPercent": 122.7, "firstAtMs": 1, "lastAtMs": 215941 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 44, "peakUsedMb": 91560, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 48889, "totalOutputTokens": 2485, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 39386, "todayTokensCached": 5214, "todayTokensOut": 2600, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 39386, "totalTokensCached": 5214, "totalTokensOut": 2600, "totalCost": 0, "medianOutputTokensPerSec": 38.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 38.4, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8068, "cachedInputTokens": 0, "outputTokens": 869, "cost": 0, "durationMs": 44038, "outputTokensPerSec": 30.20326821221714, "at": "2026-10-28T01:50:35.564Z" }, "lastUpdated": "2026-10-28T01:50:35.564Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:50:37.113Z" } }, "source": "engine-log" }, "derived": { "durationMs": 216740, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 2485, "promptTokensPerSec": null, "peakRssMb": 18250, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 39386, "output": 2600 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 547, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-schema-migration", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "schema-migration", "repetition": 1 }, "trialId": "schema-migration-qwen3-8-27b-q4-2026-10-28T01-51-02-453Z-xbcn", "success": true, "includedInCapability": true, "durationMs": 226387, "timing": { "startedAt": "2026-10-28T01:51:02.453Z", "finishedAt": "2026-10-28T01:54:48.840Z", "timeToFirstArtifactMs": 41436, "timeToLastArtifactWriteMs": 219894, "timeToFirstTokenMs": 18206, "firstTurnTtftMs": 16383, "timeToFirstToolCallMs": 20111 }, "metrics": { "process": { "sampleCount": 45, "peakRssMb": 46405, "peakCpuPercent": 120.6, "firstAtMs": 1, "lastAtMs": 220936 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 45, "peakUsedMb": 101345, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 49652, "totalOutputTokens": 3189, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 58386, "todayTokensCached": 19738, "todayTokensOut": 2907, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 58386, "totalTokensCached": 19738, "totalTokensOut": 2907, "totalCost": 0, "medianOutputTokensPerSec": 41.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 41.6, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11384, "cachedInputTokens": 10566, "outputTokens": 997, "cost": 0, "durationMs": 37496, "outputTokensPerSec": 28.71418402851353, "at": "2026-10-28T01:54:42.347Z" }, "lastUpdated": "2026-10-28T01:54:42.347Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T01:54:45.481Z" } }, "source": "engine-log" }, "derived": { "durationMs": 221981, "meanTokensPerSec": 36.85, "genTokensPerSec": 36.85, "genTokens": 3189, "promptTokensPerSec": 422.3, "peakRssMb": 46405, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 58386, "output": 2907 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-schema-migration", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "schema-migration", "repetition": 1 }, "trialId": "schema-migration-mlx-gemma4-12b-q4-2026-10-28T01-55-10-372Z-cx0a", "success": true, "includedInCapability": true, "durationMs": 405355, "timing": { "startedAt": "2026-10-28T01:55:10.373Z", "finishedAt": "2026-10-28T02:01:55.728Z", "timeToFirstArtifactMs": 40900, "timeToLastArtifactWriteMs": 403138, "timeToFirstTokenMs": 12778, "firstTurnTtftMs": 11147, "timeToFirstToolCallMs": 33688 }, "metrics": { "process": { "sampleCount": 81, "peakRssMb": 10663, "peakCpuPercent": 119.9, "firstAtMs": 0, "lastAtMs": 401850 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 81, "peakUsedMb": 73957, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 126985, "totalOutputTokens": 7425, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 121963, "todayTokensCached": 68058, "todayTokensOut": 1347, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 121963, "totalTokensCached": 68058, "totalTokensOut": 1347, "totalCost": 0, "medianOutputTokensPerSec": 39.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.8, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15872, "cachedInputTokens": 15558, "outputTokens": 42, "cost": 0, "durationMs": 80845, "outputTokensPerSec": 38.017003707448936, "at": "2026-10-28T02:01:53.511Z" }, "lastUpdated": "2026-10-28T02:01:53.511Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:01:54.320Z" } }, "source": "engine-log" }, "derived": { "durationMs": 402883, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 7425, "promptTokensPerSec": null, "peakRssMb": 10663, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 121963, "output": 1347 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1045, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-core-failing-tests-spec", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "failing-tests-spec", "repetition": 1 }, "trialId": "failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-28T02-02-17-622Z-ixp1", "success": true, "includedInCapability": true, "durationMs": 69216, "timing": { "startedAt": "2026-10-28T02:02:17.623Z", "finishedAt": "2026-10-28T02:03:26.839Z", "timeToFirstArtifactMs": 37671, "timeToLastArtifactWriteMs": 66130, "timeToFirstTokenMs": 14928, "firstTurnTtftMs": 13242, "timeToFirstToolCallMs": 16755 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 17985, "peakCpuPercent": 126.4, "firstAtMs": 0, "lastAtMs": 65268 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 91738, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 20278, "totalOutputTokens": 391, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 12926, "todayTokensCached": 1877, "todayTokensOut": 525, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 12926, "totalTokensCached": 1877, "totalTokensOut": 525, "totalCost": 0, "medianOutputTokensPerSec": 40.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 40.2, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8008, "cachedInputTokens": 1877, "outputTokens": 90, "cost": 0, "durationMs": 23909, "outputTokensPerSec": 42.35129052453445, "at": "2026-10-28T02:03:23.756Z" }, "lastUpdated": "2026-10-28T02:03:23.756Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:03:25.761Z" } }, "source": "engine-log" }, "derived": { "durationMs": 67073, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 391, "promptTokensPerSec": null, "peakRssMb": 17985, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12926, "output": 525 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 600, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-failing-tests-spec", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "failing-tests-spec", "repetition": 1 }, "trialId": "failing-tests-spec-qwen3-8-27b-q4-2026-10-28T02-03-48-290Z-xl35", "success": true, "includedInCapability": true, "durationMs": 44160, "timing": { "startedAt": "2026-10-28T02:03:48.290Z", "finishedAt": "2026-10-28T02:04:32.450Z", "timeToFirstArtifactMs": 40336, "timeToLastArtifactWriteMs": 40338, "timeToFirstTokenMs": 16491, "firstTurnTtftMs": 14649, "timeToFirstToolCallMs": 17389 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 37698, "peakCpuPercent": 110.3, "firstAtMs": 0, "lastAtMs": 40158 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 95057, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8646, "totalOutputTokens": 608, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4963, "todayTokensCached": 0, "todayTokensOut": 551, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4963, "totalTokensCached": 0, "totalTokensOut": 551, "totalCost": 0, "medianOutputTokensPerSec": 45, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 45, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4963, "outputTokens": 551, "cost": 0, "durationMs": 38496, "outputTokensPerSec": 44.963488421125085, "at": "2026-10-28T02:04:28.628Z" }, "lastUpdated": "2026-10-28T02:04:28.628Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:04:30.798Z" } }, "source": "engine-log" }, "derived": { "durationMs": 41460, "meanTokensPerSec": 45.77, "genTokensPerSec": 45.77, "genTokens": 608, "promptTokensPerSec": 467.45, "peakRssMb": 37698, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4963, "output": 551 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-failing-tests-spec", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "failing-tests-spec", "repetition": 1 }, "trialId": "failing-tests-spec-mlx-gemma4-12b-q4-2026-10-28T02-04-51-967Z-dgwy", "success": false, "includedInCapability": true, "durationMs": 201489, "timing": { "startedAt": "2026-10-28T02:04:51.968Z", "finishedAt": "2026-10-28T02:08:13.457Z", "timeToFirstArtifactMs": 60889, "timeToLastArtifactWriteMs": 198405, "timeToFirstTokenMs": 11469, "firstTurnTtftMs": 9823, "timeToFirstToolCallMs": 12011 }, "metrics": { "process": { "sampleCount": 40, "peakRssMb": 11754, "peakCpuPercent": 114.8, "firstAtMs": 0, "lastAtMs": 195811 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 76704, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 114867, "totalOutputTokens": 2055, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 73060, "todayTokensCached": 32819, "todayTokensOut": 1702, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 73060, "totalTokensCached": 32819, "totalTokensOut": 1702, "totalCost": 0, "medianOutputTokensPerSec": 41.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 41.9, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 20231, "cachedInputTokens": 19148, "outputTokens": 458, "cost": 0, "durationMs": 13737, "outputTokensPerSec": 36.15020191298372, "at": "2026-10-28T02:08:10.373Z" }, "lastUpdated": "2026-10-28T02:08:10.373Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:08:11.894Z" } }, "source": "engine-log" }, "derived": { "durationMs": 198861, "meanTokensPerSec": 43, "genTokensPerSec": 43, "genTokens": 2055, "promptTokensPerSec": null, "peakRssMb": 11754, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 73060, "output": 1702 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1300, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-failing-tests-spec", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "failing-tests-spec", "repetition": 1 }, "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T02-08-37-855Z-m1ni", "success": true, "includedInCapability": true, "durationMs": 33956, "timing": { "startedAt": "2026-10-28T02:08:37.856Z", "finishedAt": "2026-10-28T02:09:11.812Z", "timeToFirstArtifactMs": 28297, "timeToLastArtifactWriteMs": 28299, "timeToFirstTokenMs": 10423, "firstTurnTtftMs": 8512, "timeToFirstToolCallMs": 11616 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 16569, "peakCpuPercent": 100.9, "firstAtMs": 0, "lastAtMs": 30119 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 71144, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13669, "totalOutputTokens": 791, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6623, "todayTokensCached": 0, "todayTokensOut": 653, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6623, "totalTokensCached": 0, "totalTokensOut": 653, "totalCost": 0, "medianOutputTokensPerSec": 87, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 87, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6623, "outputTokens": 653, "cost": 0, "durationMs": 26388, "outputTokensPerSec": 87.00936791964605, "at": "2026-10-28T02:09:06.155Z" }, "lastUpdated": "2026-10-28T02:09:06.155Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:09:10.327Z" } }, "source": "engine-log" }, "derived": { "durationMs": 31425, "meanTokensPerSec": 88.93, "genTokensPerSec": 88.93, "genTokens": 791, "promptTokensPerSec": 999.81, "peakRssMb": 16569, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6623, "output": 653 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-core-symptom-debug", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "symptom-debug", "repetition": 1 }, "trialId": "symptom-debug-qwen3-8-27b-q4-2026-10-28T02-09-32-681Z-n9bo", "success": true, "includedInCapability": true, "durationMs": 43302, "timing": { "startedAt": "2026-10-28T02:09:32.681Z", "finishedAt": "2026-10-28T02:10:15.983Z", "timeToFirstArtifactMs": 37285, "timeToLastArtifactWriteMs": 37285, "timeToFirstTokenMs": 17130, "firstTurnTtftMs": 15294, "timeToFirstToolCallMs": 18967 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 37332, "peakCpuPercent": 93.1, "firstAtMs": 0, "lastAtMs": 40156 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 93767, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5402, "totalOutputTokens": 541, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:10:14.411Z" } }, "source": "engine-log" }, "derived": { "durationMs": 40680, "meanTokensPerSec": 32.13, "genTokensPerSec": 32.13, "genTokens": 541, "promptTokensPerSec": 452.26, "peakRssMb": 37332, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-symptom-debug", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "symptom-debug", "repetition": 1 }, "trialId": "symptom-debug-mlx-gemma4-12b-q4-2026-10-28T02-10-37-236Z-x3jd", "success": true, "includedInCapability": true, "durationMs": 248762, "timing": { "startedAt": "2026-10-28T02:10:37.236Z", "finishedAt": "2026-10-28T02:14:45.998Z", "timeToFirstArtifactMs": 243514, "timeToLastArtifactWriteMs": 243514, "timeToFirstTokenMs": 11753, "firstTurnTtftMs": 10121, "timeToFirstToolCallMs": 14667 }, "metrics": { "process": { "sampleCount": 50, "peakRssMb": 10630, "peakCpuPercent": 115.5, "firstAtMs": 0, "lastAtMs": 246090 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 50, "peakUsedMb": 74266, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 23140, "totalOutputTokens": 8181, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 23092, "todayTokensCached": 6540, "todayTokensOut": 8279, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 23092, "totalTokensCached": 6540, "totalTokensOut": 8279, "totalCost": 0, "medianOutputTokensPerSec": 39.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.6, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15629, "cachedInputTokens": 0, "outputTokens": 87, "cost": 0, "durationMs": 17605, "outputTokensPerSec": 39.56036627191046, "at": "2026-10-28T02:14:40.759Z" }, "lastUpdated": "2026-10-28T02:14:40.759Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:14:44.935Z" } }, "source": "engine-log" }, "derived": { "durationMs": 246638, "meanTokensPerSec": 42, "genTokensPerSec": 42, "genTokens": 8181, "promptTokensPerSec": null, "peakRssMb": 10630, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 23092, "output": 8279 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1165.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-symptom-debug", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "symptom-debug", "repetition": 1 }, "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T02-15-07-525Z-i2fp", "success": true, "includedInCapability": true, "durationMs": 149187, "timing": { "startedAt": "2026-10-28T02:15:07.526Z", "finishedAt": "2026-10-28T02:17:36.713Z", "timeToFirstArtifactMs": 140208, "timeToLastArtifactWriteMs": 140208, "timeToFirstTokenMs": 10471, "firstTurnTtftMs": 8627, "timeToFirstToolCallMs": 15702 }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 18187, "peakCpuPercent": 115.8, "firstAtMs": 0, "lastAtMs": 140547 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 71718, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 7236, "totalOutputTokens": 8632, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:17:29.650Z" } }, "source": "engine-log" }, "derived": { "durationMs": 141072, "meanTokensPerSec": 67.72, "genTokensPerSec": 67.72, "genTokens": 8632, "promptTokensPerSec": 910.42, "peakRssMb": 18187, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-symptom-debug", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "symptom-debug", "repetition": 1 }, "trialId": "symptom-debug-mlx-qwen3-8-27b-q4-2026-10-28T02-17-58-089Z-s8hd", "success": true, "includedInCapability": true, "durationMs": 37867, "timing": { "startedAt": "2026-10-28T02:17:58.089Z", "finishedAt": "2026-10-28T02:18:35.956Z", "timeToFirstArtifactMs": 33315, "timeToLastArtifactWriteMs": 33315, "timeToFirstTokenMs": 15490, "firstTurnTtftMs": 13847, "timeToFirstToolCallMs": 17090 }, "metrics": { "process": { "sampleCount": 8, "peakRssMb": 17963, "peakCpuPercent": 122.1, "firstAtMs": 1, "lastAtMs": 35142 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 89999, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5388, "totalOutputTokens": 324, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:18:34.830Z" } }, "source": "engine-log" }, "derived": { "durationMs": 35668, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 324, "promptTokensPerSec": null, "peakRssMb": 17963, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 564, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-core-data-wrangle", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "data-wrangle", "repetition": 1 }, "trialId": "data-wrangle-mlx-gemma4-12b-q4-2026-10-28T02-18-56-828Z-7owd", "success": false, "includedInCapability": true, "durationMs": 274346, "timing": { "startedAt": "2026-10-28T02:18:56.829Z", "finishedAt": "2026-10-28T02:23:31.175Z", "timeToFirstArtifactMs": 54853, "timeToLastArtifactWriteMs": 270385, "timeToFirstTokenMs": 11640, "firstTurnTtftMs": 10007, "timeToFirstToolCallMs": 25106 }, "metrics": { "process": { "sampleCount": 55, "peakRssMb": 10604, "peakCpuPercent": 98.3, "firstAtMs": 0, "lastAtMs": 271197 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 55, "peakUsedMb": 74439, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 63123, "totalOutputTokens": 6845, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 61022, "todayTokensCached": 11992, "todayTokensOut": 2740, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 61022, "totalTokensCached": 11992, "totalTokensOut": 2740, "totalCost": 0, "medianOutputTokensPerSec": 38.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.3, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18389, "cachedInputTokens": 0, "outputTokens": 1215, "cost": 0, "durationMs": 52743, "outputTokensPerSec": 34.17226297581007, "at": "2026-10-28T02:23:27.214Z" }, "lastUpdated": "2026-10-28T02:23:27.214Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:23:29.635Z" } }, "source": "engine-log" }, "derived": { "durationMs": 271742, "meanTokensPerSec": 40, "genTokensPerSec": 40, "genTokens": 6845, "promptTokensPerSec": null, "peakRssMb": 10604, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 61022, "output": 2740 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1207.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-data-wrangle", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "data-wrangle", "repetition": 1 }, "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T02-23-53-668Z-mu0d", "success": true, "includedInCapability": true, "durationMs": 71942, "timing": { "startedAt": "2026-10-28T02:23:53.669Z", "finishedAt": "2026-10-28T02:25:05.611Z", "timeToFirstArtifactMs": 38087, "timeToLastArtifactWriteMs": 60639, "timeToFirstTokenMs": 10695, "firstTurnTtftMs": 8863, "timeToFirstToolCallMs": 15624 }, "metrics": { "process": { "sampleCount": 13, "peakRssMb": 19940, "peakCpuPercent": 105.8, "firstAtMs": 1, "lastAtMs": 60237 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 74046, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 10685, "totalOutputTokens": 3466, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 10657, "todayTokensCached": 10605, "todayTokensOut": 205, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 10657, "totalTokensCached": 10605, "totalTokensOut": 205, "totalCost": 0, "medianOutputTokensPerSec": 64.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 64.5, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10657, "cachedInputTokens": 10605, "outputTokens": 205, "cost": 0, "durationMs": 58806, "outputTokensPerSec": 64.46591412192211, "at": "2026-10-28T02:24:54.307Z" }, "lastUpdated": "2026-10-28T02:24:54.307Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:24:55.502Z" } }, "source": "engine-log" }, "derived": { "durationMs": 60783, "meanTokensPerSec": 82.66, "genTokensPerSec": 82.66, "genTokens": 3466, "promptTokensPerSec": 836.84, "peakRssMb": 19940, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 10657, "output": 205 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-data-wrangle", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "data-wrangle", "repetition": 1 }, "trialId": "data-wrangle-mlx-qwen3-8-27b-q4-2026-10-28T02-25-27-434Z-v934", "success": true, "includedInCapability": true, "durationMs": 479874, "timing": { "startedAt": "2026-10-28T02:25:27.434Z", "finishedAt": "2026-10-28T02:33:27.308Z", "timeToFirstArtifactMs": 66625, "timeToLastArtifactWriteMs": 471137, "timeToFirstTokenMs": 15588, "firstTurnTtftMs": 13927, "timeToFirstToolCallMs": 16954 }, "metrics": { "process": { "sampleCount": 96, "peakRssMb": 17999, "peakCpuPercent": 126.5, "firstAtMs": 0, "lastAtMs": 476890 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 96, "peakUsedMb": 90441, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13978, "totalOutputTokens": 9694, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:33:25.922Z" } }, "source": "engine-log" }, "derived": { "durationMs": 477417, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 9694, "promptTokensPerSec": null, "peakRssMb": 17999, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 598, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-data-wrangle", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "data-wrangle", "repetition": 1 }, "trialId": "data-wrangle-qwen3-8-27b-q4-2026-10-28T02-33-50-542Z-h65s", "success": true, "includedInCapability": true, "durationMs": 96484, "timing": { "startedAt": "2026-10-28T02:33:50.543Z", "finishedAt": "2026-10-28T02:35:27.027Z", "timeToFirstArtifactMs": 60821, "timeToLastArtifactWriteMs": 82893, "timeToFirstTokenMs": 18459, "firstTurnTtftMs": 16638, "timeToFirstToolCallMs": 20493 }, "metrics": { "process": { "sampleCount": 19, "peakRssMb": 38388, "peakCpuPercent": 113.1, "firstAtMs": 0, "lastAtMs": 90347 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 19, "peakUsedMb": 94032, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8561, "totalOutputTokens": 1948, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:35:22.460Z" } }, "source": "engine-log" }, "derived": { "durationMs": 90872, "meanTokensPerSec": 32.98, "genTokensPerSec": 32.98, "genTokens": 1948, "promptTokensPerSec": 380.44, "peakRssMb": 38388, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-core-incident-postmortem", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "incident-postmortem", "repetition": 1 }, "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T02-35-48-418Z-0ut3", "success": true, "includedInCapability": true, "durationMs": 114998, "timing": { "startedAt": "2026-10-28T02:35:48.418Z", "finishedAt": "2026-10-28T02:37:43.416Z", "timeToFirstArtifactMs": 80671, "timeToLastArtifactWriteMs": 109125, "timeToFirstTokenMs": 15048, "firstTurnTtftMs": 13159, "timeToFirstToolCallMs": 18888 }, "metrics": { "process": { "sampleCount": 23, "peakRssMb": 22192, "peakCpuPercent": 112.5, "firstAtMs": 0, "lastAtMs": 110456 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 76269, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 40153, "totalOutputTokens": 3958, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 28211, "todayTokensCached": 17744, "todayTokensOut": 933, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 28211, "totalTokensCached": 17744, "totalTokensOut": 933, "totalCost": 0, "medianOutputTokensPerSec": 57.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 57.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9198, "outputTokens": 850, "cost": 0, "durationMs": 22078, "outputTokensPerSec": 60.958657823903714, "at": "2026-10-28T02:37:37.543Z" }, "lastUpdated": "2026-10-28T02:37:37.543Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:37:40.473Z" } }, "source": "engine-log" }, "derived": { "durationMs": 111004, "meanTokensPerSec": 64.81, "genTokensPerSec": 64.81, "genTokens": 3958, "promptTokensPerSec": 1040.82, "peakRssMb": 22192, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28211, "output": 933 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-incident-postmortem", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "incident-postmortem", "repetition": 1 }, "trialId": "incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-28T02-38-04-781Z-frfy", "success": true, "includedInCapability": true, "durationMs": 1007384, "timing": { "startedAt": "2026-10-28T02:38:04.781Z", "finishedAt": "2026-10-28T02:54:52.165Z", "timeToFirstArtifactMs": 854191, "timeToLastArtifactWriteMs": 1001537, "timeToFirstTokenMs": 28066, "firstTurnTtftMs": 26419, "timeToFirstToolCallMs": 36106 }, "metrics": { "process": { "sampleCount": 201, "peakRssMb": 18029, "peakCpuPercent": 122.9, "firstAtMs": 0, "lastAtMs": 1003980 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 201, "peakUsedMb": 91259, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44981, "totalOutputTokens": 22237, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 33556, "todayTokensCached": 17656, "todayTokensOut": 2316, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 33556, "totalTokensCached": 17656, "totalTokensOut": 2316, "totalCost": 0, "medianOutputTokensPerSec": 26.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.8, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 15824, "cachedInputTokens": 0, "outputTokens": 1566, "cost": 0, "durationMs": 104914, "outputTokensPerSec": 31.177241171745663, "at": "2026-10-28T02:54:46.321Z" }, "lastUpdated": "2026-10-28T02:54:46.321Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T02:54:50.375Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1004528, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 22237, "promptTokensPerSec": null, "peakRssMb": 18029, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 33556, "output": 2316 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 549.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-incident-postmortem", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "incident-postmortem", "repetition": 1 }, "trialId": "incident-postmortem-qwen3-8-27b-q4-2026-10-28T02-55-15-848Z-vq7o", "success": true, "includedInCapability": true, "durationMs": 813879, "timing": { "startedAt": "2026-10-28T02:55:15.848Z", "finishedAt": "2026-10-28T03:08:49.727Z", "timeToFirstArtifactMs": 651598, "timeToLastArtifactWriteMs": 807238, "timeToFirstTokenMs": 34641, "firstTurnTtftMs": 32811, "timeToFirstToolCallMs": 39647 }, "metrics": { "process": { "sampleCount": 162, "peakRssMb": 43212, "peakCpuPercent": 119.5, "firstAtMs": 0, "lastAtMs": 808143 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 162, "peakUsedMb": 98641, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 64641, "totalOutputTokens": 17223, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 62155, "todayTokensCached": 49913, "todayTokensOut": 1561, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 62155, "totalTokensCached": 49913, "totalTokensOut": 1561, "totalCost": 0, "medianOutputTokensPerSec": 33.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.8, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 17265, "cachedInputTokens": 10873, "outputTokens": 965, "cost": 0, "durationMs": 43172, "outputTokensPerSec": 36.345040206512216, "at": "2026-10-28T03:08:43.089Z" }, "lastUpdated": "2026-10-28T03:08:43.089Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:08:45.590Z" } }, "source": "engine-log" }, "derived": { "durationMs": 808695, "meanTokensPerSec": 27.73, "genTokensPerSec": 27.73, "genTokens": 17223, "promptTokensPerSec": 375.81, "peakRssMb": 43212, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 62155, "output": 1561 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-incident-postmortem", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "incident-postmortem", "repetition": 1 }, "trialId": "incident-postmortem-mlx-gemma4-12b-q4-2026-10-28T03-09-10-894Z-r7ig", "success": true, "includedInCapability": true, "durationMs": 243986, "timing": { "startedAt": "2026-10-28T03:09:10.895Z", "finishedAt": "2026-10-28T03:13:14.881Z", "timeToFirstArtifactMs": 166836, "timeToLastArtifactWriteMs": 240923, "timeToFirstTokenMs": 17490, "firstTurnTtftMs": 15863, "timeToFirstToolCallMs": 65163 }, "metrics": { "process": { "sampleCount": 49, "peakRssMb": 10797, "peakCpuPercent": 121.1, "firstAtMs": 0, "lastAtMs": 241080 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 75060, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 34015, "totalOutputTokens": 6589, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 66784, "todayTokensCached": 50422, "todayTokensOut": 1977, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 66784, "totalTokensCached": 50422, "totalTokensOut": 1977, "totalCost": 0, "medianOutputTokensPerSec": 36.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.8, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15170, "cachedInputTokens": 14252, "outputTokens": 476, "cost": 0, "durationMs": 13792, "outputTokensPerSec": 36.633328474111224, "at": "2026-10-28T03:13:11.818Z" }, "lastUpdated": "2026-10-28T03:13:11.818Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:13:13.585Z" } }, "source": "engine-log" }, "derived": { "durationMs": 241628, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 6589, "promptTokensPerSec": null, "peakRssMb": 10797, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 66784, "output": 1977 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1113, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-core-ops-runbook-anomaly", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 1 }, "trialId": "ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-28T03-13-38-684Z-ggrp", "success": true, "includedInCapability": true, "durationMs": 103060, "timing": { "startedAt": "2026-10-28T03:13:38.685Z", "finishedAt": "2026-10-28T03:15:21.745Z", "timeToFirstArtifactMs": 81976, "timeToLastArtifactWriteMs": 98886, "timeToFirstTokenMs": 27591, "firstTurnTtftMs": 25936, "timeToFirstToolCallMs": 28734 }, "metrics": { "process": { "sampleCount": 21, "peakRssMb": 17976, "peakCpuPercent": 110.4, "firstAtMs": 0, "lastAtMs": 100396 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 91240, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12178, "totalOutputTokens": 1953, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12130, "todayTokensCached": 12081, "todayTokensOut": 496, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12130, "totalTokensCached": 12081, "totalTokensOut": 496, "totalCost": 0, "medianOutputTokensPerSec": 30, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12130, "cachedInputTokens": 12081, "outputTokens": 496, "cost": 0, "durationMs": 97231, "outputTokensPerSec": 29.98588270859912, "at": "2026-10-28T03:15:17.571Z" }, "lastUpdated": "2026-10-28T03:15:17.571Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:15:20.696Z" } }, "source": "engine-log" }, "derived": { "durationMs": 100944, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 1953, "promptTokensPerSec": null, "peakRssMb": 17976, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12130, "output": 496 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 570, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-ops-runbook-anomaly", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 1 }, "trialId": "ops-runbook-anomaly-qwen3-8-27b-q4-2026-10-28T03-15-45-182Z-7x93", "success": true, "includedInCapability": true, "durationMs": 124103, "timing": { "startedAt": "2026-10-28T03:15:45.182Z", "finishedAt": "2026-10-28T03:17:49.285Z", "timeToFirstArtifactMs": 46513, "timeToLastArtifactWriteMs": 118670, "timeToFirstTokenMs": 32123, "firstTurnTtftMs": 30265, "timeToFirstToolCallMs": 32947 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 41653, "peakCpuPercent": 117.3, "firstAtMs": 1, "lastAtMs": 120463 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 97019, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 28302, "totalOutputTokens": 1643, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18442, "todayTokensCached": 12443, "todayTokensOut": 451, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18442, "totalTokensCached": 12443, "totalTokensOut": 451, "totalCost": 0, "medianOutputTokensPerSec": 41.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 41.7, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5667, "outputTokens": 97, "cost": 0, "durationMs": 36684, "outputTokensPerSec": 50.696522693823105, "at": "2026-10-28T03:17:43.852Z" }, "lastUpdated": "2026-10-28T03:17:43.852Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:17:47.245Z" } }, "source": "engine-log" }, "derived": { "durationMs": 121013, "meanTokensPerSec": 35.85, "genTokensPerSec": 35.85, "genTokens": 1643, "promptTokensPerSec": 448.19, "peakRssMb": 41653, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18442, "output": 451 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-ops-runbook-anomaly", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 1 }, "trialId": "ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-28T03-18-10-442Z-tjt4", "success": true, "includedInCapability": true, "durationMs": 93483, "timing": { "startedAt": "2026-10-28T03:18:10.442Z", "finishedAt": "2026-10-28T03:19:43.925Z", "timeToFirstArtifactMs": 27496, "timeToLastArtifactWriteMs": 89853, "timeToFirstTokenMs": 16601, "firstTurnTtftMs": 14959, "timeToFirstToolCallMs": 25143 }, "metrics": { "process": { "sampleCount": 19, "peakRssMb": 10617, "peakCpuPercent": 100.9, "firstAtMs": 0, "lastAtMs": 90393 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 19, "peakUsedMb": 74554, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47415, "totalOutputTokens": 222, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 26930, "todayTokensCached": 18326, "todayTokensOut": 268, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 26930, "totalTokensCached": 18326, "totalTokensOut": 268, "totalCost": 0, "medianOutputTokensPerSec": 39.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.2, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8324, "cachedInputTokens": 0, "outputTokens": 108, "cost": 0, "durationMs": 28415, "outputTokensPerSec": 43.64713265482906, "at": "2026-10-28T03:19:40.295Z" }, "lastUpdated": "2026-10-28T03:19:40.295Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:19:42.450Z" } }, "source": "engine-log" }, "derived": { "durationMs": 90943, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 222, "promptTokensPerSec": null, "peakRssMb": 10617, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 26930, "output": 268 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1413, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-ops-runbook-anomaly", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 1 }, "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T03-20-04-860Z-3xtb", "success": true, "includedInCapability": true, "durationMs": 79840, "timing": { "startedAt": "2026-10-28T03:20:04.860Z", "finishedAt": "2026-10-28T03:21:24.700Z", "timeToFirstArtifactMs": 21444, "timeToLastArtifactWriteMs": 75149, "timeToFirstTokenMs": 14831, "firstTurnTtftMs": 12963, "timeToFirstToolCallMs": 19606 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 28053, "peakCpuPercent": 104.4, "firstAtMs": 0, "lastAtMs": 75333 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 82571, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 54739, "totalOutputTokens": 1165, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 26838, "todayTokensCached": 11348, "todayTokensOut": 299, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 26838, "totalTokensCached": 11348, "totalTokensOut": 299, "totalCost": 0, "medianOutputTokensPerSec": 93.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 93.5, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8299, "outputTokens": 131, "cost": 0, "durationMs": 23740, "outputTokensPerSec": 93.5158282732878, "at": "2026-10-28T03:21:20.009Z" }, "lastUpdated": "2026-10-28T03:21:20.009Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:21:21.793Z" } }, "source": "engine-log" }, "derived": { "durationMs": 75882, "meanTokensPerSec": 85.25, "genTokensPerSec": 85.25, "genTokens": 1165, "promptTokensPerSec": 1066.16, "peakRssMb": 28053, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 26838, "output": 299 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-core-plan-and-estimate", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-qwen3-8-27b-q4-2026-10-28T03-21-46-450Z-z615", "success": true, "includedInCapability": true, "durationMs": 80654, "timing": { "startedAt": "2026-10-28T03:21:46.450Z", "finishedAt": "2026-10-28T03:23:07.104Z", "timeToFirstArtifactMs": 72441, "timeToLastArtifactWriteMs": 72441, "timeToFirstTokenMs": 17706, "firstTurnTtftMs": 15955, "timeToFirstToolCallMs": 18971 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 37041, "peakCpuPercent": 106.8, "firstAtMs": 0, "lastAtMs": 75291 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 94377, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4876, "totalOutputTokens": 1363, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:23:03.316Z" } }, "source": "engine-log" }, "derived": { "durationMs": 75817, "meanTokensPerSec": 24.74, "genTokensPerSec": 24.74, "genTokens": 1363, "promptTokensPerSec": 440.75, "peakRssMb": 37041, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-core-plan-and-estimate", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T03-23-29-007Z-psno", "success": true, "includedInCapability": true, "durationMs": 37717, "timing": { "startedAt": "2026-10-28T03:23:29.008Z", "finishedAt": "2026-10-28T03:24:06.725Z", "timeToFirstArtifactMs": 34212, "timeToLastArtifactWriteMs": 34212, "timeToFirstTokenMs": 12317, "firstTurnTtftMs": 10689, "timeToFirstToolCallMs": 21102 }, "metrics": { "process": { "sampleCount": 8, "peakRssMb": 10704, "peakCpuPercent": 117.6, "firstAtMs": 0, "lastAtMs": 35139 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 74494, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5729, "totalOutputTokens": 621, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:24:05.739Z" } }, "source": "engine-log" }, "derived": { "durationMs": 35665, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 621, "promptTokensPerSec": null, "peakRssMb": 10704, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1201, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-plan-and-estimate", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T03-24-28-463Z-kamt", "success": true, "includedInCapability": true, "durationMs": 31116, "timing": { "startedAt": "2026-10-28T03:24:28.464Z", "finishedAt": "2026-10-28T03:24:59.580Z", "timeToFirstArtifactMs": 22301, "timeToLastArtifactWriteMs": 23253, "timeToFirstTokenMs": 10076, "firstTurnTtftMs": 8232, "timeToFirstToolCallMs": 14755 }, "metrics": { "process": { "sampleCount": 6, "peakRssMb": 14882, "peakCpuPercent": 90.6, "firstAtMs": 0, "lastAtMs": 25099 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 69566, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6151, "totalOutputTokens": 941, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6148, "todayTokensCached": 5650, "todayTokensOut": 26, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6148, "totalTokensCached": 5650, "totalTokensOut": 26, "totalCost": 0, "medianOutputTokensPerSec": 72.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 72.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6148, "cachedInputTokens": 5650, "outputTokens": 26, "cost": 0, "durationMs": 21409, "outputTokensPerSec": 72.90287587264743, "at": "2026-10-28T03:24:51.717Z" }, "lastUpdated": "2026-10-28T03:24:51.717Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:24:55.146Z" } }, "source": "engine-log" }, "derived": { "durationMs": 25634, "meanTokensPerSec": 78.51, "genTokensPerSec": 78.51, "genTokens": 941, "promptTokensPerSec": 1076.1, "peakRssMb": 14882, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6148, "output": 26 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-plan-and-estimate", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T03-25-19-783Z-ky1v", "success": true, "includedInCapability": true, "durationMs": 67910, "timing": { "startedAt": "2026-10-28T03:25:19.784Z", "finishedAt": "2026-10-28T03:26:27.694Z", "timeToFirstArtifactMs": 66042, "timeToLastArtifactWriteMs": 66042, "timeToFirstTokenMs": 14999, "firstTurnTtftMs": 13348, "timeToFirstToolCallMs": 16000 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 18153, "peakCpuPercent": 130.6, "firstAtMs": 0, "lastAtMs": 65254 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 90424, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4818, "totalOutputTokens": 1548, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:26:26.628Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65777, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 1548, "promptTokensPerSec": null, "peakRssMb": 18153, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 658, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-core-conflict-synthesis", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T03-26-49-015Z-byen", "success": true, "includedInCapability": true, "durationMs": 73437, "timing": { "startedAt": "2026-10-28T03:26:49.015Z", "finishedAt": "2026-10-28T03:28:02.452Z", "timeToFirstArtifactMs": 38020, "timeToLastArtifactWriteMs": 66481, "timeToFirstTokenMs": 15443, "firstTurnTtftMs": 13807, "timeToFirstToolCallMs": 27105 }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 10707, "peakCpuPercent": 119.3, "firstAtMs": 0, "lastAtMs": 70306 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 74717, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 24283, "totalOutputTokens": 1322, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 17802, "todayTokensCached": 10640, "todayTokensOut": 485, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 17802, "totalTokensCached": 10640, "totalTokensOut": 485, "totalCost": 0, "medianOutputTokensPerSec": 43.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 43.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6692, "cachedInputTokens": 0, "outputTokens": 462, "cost": 0, "durationMs": 24651, "outputTokensPerSec": 43.21033191904962, "at": "2026-10-28T03:27:55.500Z" }, "lastUpdated": "2026-10-28T03:27:55.500Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:28:00.935Z" } }, "source": "engine-log" }, "derived": { "durationMs": 70856, "meanTokensPerSec": 44, "genTokensPerSec": 44, "genTokens": 1322, "promptTokensPerSec": null, "peakRssMb": 10707, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 17802, "output": 485 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1460.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-core-conflict-synthesis", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T03-28-24-785Z-r0z2", "success": true, "includedInCapability": true, "durationMs": 185728, "timing": { "startedAt": "2026-10-28T03:28:24.785Z", "finishedAt": "2026-10-28T03:31:30.513Z", "timeToFirstArtifactMs": 24478, "timeToLastArtifactWriteMs": 181795, "timeToFirstTokenMs": 14183, "firstTurnTtftMs": 12327, "timeToFirstToolCallMs": 18683 }, "metrics": { "process": { "sampleCount": 37, "peakRssMb": 24776, "peakCpuPercent": 95.5, "firstAtMs": 0, "lastAtMs": 180756 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 37, "peakUsedMb": 78902, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 143560, "totalOutputTokens": 2531, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 35810, "todayTokensCached": 17132, "todayTokensOut": 1501, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 35810, "totalTokensCached": 17132, "totalTokensOut": 1501, "totalCost": 0, "medianOutputTokensPerSec": 79.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 79.8, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9937, "outputTokens": 460, "cost": 0, "durationMs": 65192, "outputTokensPerSec": 79.7271975004221, "at": "2026-10-28T03:31:26.580Z" }, "lastUpdated": "2026-10-28T03:31:26.580Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:31:27.119Z" } }, "source": "engine-log" }, "derived": { "durationMs": 181285, "meanTokensPerSec": 82.42, "genTokensPerSec": 82.42, "genTokens": 2531, "promptTokensPerSec": 1066.59, "peakRssMb": 24776, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35810, "output": 1501 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-core-conflict-synthesis", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T03-31-52-036Z-1i5n", "success": true, "includedInCapability": true, "durationMs": 1240132, "timing": { "startedAt": "2026-10-28T03:31:52.036Z", "finishedAt": "2026-10-28T03:52:32.168Z", "timeToFirstArtifactMs": 281837, "timeToLastArtifactWriteMs": 1234139, "timeToFirstTokenMs": 30562, "firstTurnTtftMs": 28909, "timeToFirstToolCallMs": 32994 }, "metrics": { "process": { "sampleCount": 247, "peakRssMb": 18386, "peakCpuPercent": 124.6, "firstAtMs": 1, "lastAtMs": 1236589 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 247, "peakUsedMb": 90643, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 93557, "totalOutputTokens": 14345, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 47762, "todayTokensCached": 15989, "todayTokensOut": 3156, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 47762, "totalTokensCached": 15989, "totalTokensOut": 3156, "totalCost": 0, "medianOutputTokensPerSec": 32.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.6, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9848, "cachedInputTokens": 0, "outputTokens": 1318, "cost": 0, "durationMs": 110032, "outputTokensPerSec": 32.907460426405514, "at": "2026-10-28T03:41:09.950Z" }, "lastUpdated": "2026-10-28T03:41:09.950Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T03:52:30.228Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1237119, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 14345, "promptTokensPerSec": null, "peakRssMb": 18386, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 47762, "output": 3156 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 508, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-core-conflict-synthesis", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-qwen3-8-27b-q4-2026-10-28T03-52-53-902Z-1ik5", "success": true, "includedInCapability": true, "durationMs": 506118, "timing": { "startedAt": "2026-10-28T03:52:53.902Z", "finishedAt": "2026-10-28T04:01:20.020Z", "timeToFirstArtifactMs": 160611, "timeToLastArtifactWriteMs": 501131, "timeToFirstTokenMs": 32221, "firstTurnTtftMs": 30395, "timeToFirstToolCallMs": 34066 }, "metrics": { "process": { "sampleCount": 101, "peakRssMb": 48363, "peakCpuPercent": 107.6, "firstAtMs": 0, "lastAtMs": 501990 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 101, "peakUsedMb": 103786, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 105352, "totalOutputTokens": 6820, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 30025, "todayTokensCached": 22035, "todayTokensOut": 1303, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 30025, "totalTokensCached": 22035, "totalTokensOut": 1303, "totalCost": 0, "medianOutputTokensPerSec": 23.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.3, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7850, "outputTokens": 806, "cost": 0, "durationMs": 133569, "outputTokensPerSec": 38.99892614882358, "at": "2026-10-28T04:01:15.033Z" }, "lastUpdated": "2026-10-28T04:01:15.033Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:01:17.486Z" } }, "source": "engine-log" }, "derived": { "durationMs": 502537, "meanTokensPerSec": 30.79, "genTokensPerSec": 30.79, "genTokens": 6820, "promptTokensPerSec": 398.1, "peakRssMb": 48363, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 30025, "output": 1303 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-productivity-constrained-comms", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "constrained-comms", "repetition": 1 }, "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T04-01-42-817Z-h3aq", "success": true, "includedInCapability": true, "durationMs": 33850, "timing": { "startedAt": "2026-10-28T04:01:42.817Z", "finishedAt": "2026-10-28T04:02:16.667Z", "timeToFirstArtifactMs": 28405, "timeToLastArtifactWriteMs": 28407, "timeToFirstTokenMs": 15570, "firstTurnTtftMs": 13737, "timeToFirstToolCallMs": 19165 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 15195, "peakCpuPercent": 94.3, "firstAtMs": 1, "lastAtMs": 30120 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 70190, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15528, "totalOutputTokens": 536, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5605, "todayTokensCached": 0, "todayTokensOut": 227, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5605, "totalTokensCached": 0, "totalTokensOut": 227, "totalCost": 0, "medianOutputTokensPerSec": 64.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 64.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5605, "outputTokens": 227, "cost": 0, "durationMs": 26574, "outputTokensPerSec": 64.90422032512994, "at": "2026-10-28T04:02:11.224Z" }, "lastUpdated": "2026-10-28T04:02:11.224Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:02:14.536Z" } }, "source": "engine-log" }, "derived": { "durationMs": 30670, "meanTokensPerSec": 75.49, "genTokensPerSec": 75.49, "genTokens": 536, "promptTokensPerSec": 990.53, "peakRssMb": 15195, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5605, "output": 227 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-constrained-comms", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "constrained-comms", "repetition": 1 }, "trialId": "constrained-comms-mlx-qwen3-8-27b-q4-2026-10-28T04-02-37-132Z-iyzn", "success": false, "includedInCapability": true, "durationMs": 88301, "timing": { "startedAt": "2026-10-28T04:02:37.132Z", "finishedAt": "2026-10-28T04:04:05.433Z", "timeToFirstArtifactMs": 36158, "timeToLastArtifactWriteMs": 83105, "timeToFirstTokenMs": 25927, "firstTurnTtftMs": 24286, "timeToFirstToolCallMs": 27427 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 18176, "peakCpuPercent": 123, "firstAtMs": 1, "lastAtMs": 85359 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 92351, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 34169, "totalOutputTokens": 7, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 16717, "todayTokensCached": 1391, "todayTokensOut": 670, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 16717, "totalTokensCached": 1391, "totalTokensOut": 670, "totalCost": 0, "medianOutputTokensPerSec": 45.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 45.3, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4721, "cachedInputTokens": 0, "outputTokens": 207, "cost": 0, "durationMs": 11662, "outputTokensPerSec": 45.39174703569371, "at": "2026-10-28T04:04:00.237Z" }, "lastUpdated": "2026-10-28T04:04:00.237Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:04:04.108Z" } }, "source": "engine-log" }, "derived": { "durationMs": 85911, "meanTokensPerSec": 397.81, "genTokensPerSec": null, "genTokens": 7, "promptTokensPerSec": null, "peakRssMb": 18176, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 16717, "output": 670 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 693, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-constrained-comms", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "constrained-comms", "repetition": 1 }, "trialId": "constrained-comms-qwen3-8-27b-q4-2026-10-28T04-04-26-886Z-eont", "success": false, "includedInCapability": true, "durationMs": 94173, "timing": { "startedAt": "2026-10-28T04:04:26.886Z", "finishedAt": "2026-10-28T04:06:01.059Z", "timeToFirstArtifactMs": 44742, "timeToLastArtifactWriteMs": 90554, "timeToFirstTokenMs": 31962, "firstTurnTtftMs": 30137, "timeToFirstToolCallMs": 32873 }, "metrics": { "process": { "sampleCount": 19, "peakRssMb": 41784, "peakCpuPercent": 107.4, "firstAtMs": 0, "lastAtMs": 90405 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 19, "peakUsedMb": 97349, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 26687, "totalOutputTokens": 843, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20780, "todayTokensCached": 11676, "todayTokensOut": 729, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20780, "totalTokensCached": 11676, "totalTokensOut": 729, "totalCost": 0, "medianOutputTokensPerSec": 46.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 46.4, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5233, "outputTokens": 210, "cost": 0, "durationMs": 14091, "outputTokensPerSec": 49.78277796952485, "at": "2026-10-28T04:05:57.439Z" }, "lastUpdated": "2026-10-28T04:05:57.439Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:05:58.890Z" } }, "source": "engine-log" }, "derived": { "durationMs": 90954, "meanTokensPerSec": 47.04, "genTokensPerSec": 47.04, "genTokens": 843, "promptTokensPerSec": 498.81, "peakRssMb": 41784, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20780, "output": 729 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-constrained-comms", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "constrained-comms", "repetition": 1 }, "trialId": "constrained-comms-mlx-gemma4-12b-q4-2026-10-28T04-06-22-235Z-ukrp", "success": true, "includedInCapability": true, "durationMs": 63085, "timing": { "startedAt": "2026-10-28T04:06:22.235Z", "finishedAt": "2026-10-28T04:07:25.320Z", "timeToFirstArtifactMs": 40045, "timeToLastArtifactWriteMs": 56972, "timeToFirstTokenMs": 16142, "firstTurnTtftMs": 14516, "timeToFirstToolCallMs": 30530 }, "metrics": { "process": { "sampleCount": 13, "peakRssMb": 10577, "peakCpuPercent": 101.9, "firstAtMs": 0, "lastAtMs": 60266 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 74673, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 26764, "totalOutputTokens": 612, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 11069, "todayTokensCached": 0, "todayTokensOut": 421, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 11069, "totalTokensCached": 0, "totalTokensOut": 421, "totalCost": 0, "medianOutputTokensPerSec": 41, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 41, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5519, "cachedInputTokens": 0, "outputTokens": 237, "cost": 0, "durationMs": 15159, "outputTokensPerSec": 40.91620839416529, "at": "2026-10-28T04:07:19.211Z" }, "lastUpdated": "2026-10-28T04:07:19.211Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:07:24.116Z" } }, "source": "engine-log" }, "derived": { "durationMs": 60817, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 612, "promptTokensPerSec": null, "peakRssMb": 10577, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11069, "output": 421 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1295, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-productivity-records-intake", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "records-intake", "repetition": 1 }, "trialId": "records-intake-mlx-qwen3-8-27b-q4-2026-10-28T04-07-46-716Z-w948", "success": true, "includedInCapability": true, "durationMs": 77827, "timing": { "startedAt": "2026-10-28T04:07:46.717Z", "finishedAt": "2026-10-28T04:09:04.544Z", "timeToFirstArtifactMs": 72121, "timeToLastArtifactWriteMs": 72121, "timeToFirstTokenMs": 15823, "firstTurnTtftMs": 14183, "timeToFirstToolCallMs": 16583 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 18013, "peakCpuPercent": 126.8, "firstAtMs": 0, "lastAtMs": 75306 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 90695, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5562, "totalOutputTokens": 1823, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:09:03.620Z" } }, "source": "engine-log" }, "derived": { "durationMs": 75832, "meanTokensPerSec": 42, "genTokensPerSec": 42, "genTokens": 1823, "promptTokensPerSec": null, "peakRssMb": 18013, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 598, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-records-intake", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "records-intake", "repetition": 1 }, "trialId": "records-intake-qwen3-8-27b-q4-2026-10-28T04-09-26-351Z-flxy", "success": true, "includedInCapability": true, "durationMs": 140466, "timing": { "startedAt": "2026-10-28T04:09:26.352Z", "finishedAt": "2026-10-28T04:11:46.818Z", "timeToFirstArtifactMs": 133438, "timeToLastArtifactWriteMs": 133438, "timeToFirstTokenMs": 17675, "firstTurnTtftMs": 15859, "timeToFirstToolCallMs": 19329 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 37391, "peakCpuPercent": 117.7, "firstAtMs": 1, "lastAtMs": 135532 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 94510, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5756, "totalOutputTokens": 4065, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:11:43.461Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136058, "meanTokensPerSec": 36.25, "genTokensPerSec": 36.25, "genTokens": 4065, "promptTokensPerSec": 450.56, "peakRssMb": 37391, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-records-intake", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "records-intake", "repetition": 1 }, "trialId": "records-intake-mlx-gemma4-12b-q4-2026-10-28T04-12-08-219Z-qnj8", "success": true, "includedInCapability": true, "durationMs": 173500, "timing": { "startedAt": "2026-10-28T04:12:08.219Z", "finishedAt": "2026-10-28T04:15:01.719Z", "timeToFirstArtifactMs": 64088, "timeToLastArtifactWriteMs": 167947, "timeToFirstTokenMs": 12486, "firstTurnTtftMs": 10860, "timeToFirstToolCallMs": 28777 }, "metrics": { "process": { "sampleCount": 35, "peakRssMb": 10315, "peakCpuPercent": 110.5, "firstAtMs": 0, "lastAtMs": 170815 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 35, "peakUsedMb": 74333, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16931, "totalOutputTokens": 4624, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7567, "todayTokensCached": 6335, "todayTokensOut": 715, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7567, "totalTokensCached": 6335, "totalTokensOut": 715, "totalCost": 0, "medianOutputTokensPerSec": 37.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7567, "cachedInputTokens": 6335, "outputTokens": 715, "cost": 0, "durationMs": 83327, "outputTokensPerSec": 37.184204661937855, "at": "2026-10-28T04:13:33.172Z" }, "lastUpdated": "2026-10-28T04:13:33.172Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:15:00.631Z" } }, "source": "engine-log" }, "derived": { "durationMs": 171346, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 4624, "promptTokensPerSec": null, "peakRssMb": 10315, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 7567, "output": 715 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1146, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-records-intake", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "records-intake", "repetition": 1 }, "trialId": "records-intake-gemma4-12b-q4-2026-10-28T04-15-24-172Z-l4qq", "success": true, "includedInCapability": true, "durationMs": 125979, "timing": { "startedAt": "2026-10-28T04:15:24.172Z", "finishedAt": "2026-10-28T04:17:30.151Z", "timeToFirstArtifactMs": 47221, "timeToLastArtifactWriteMs": 121285, "timeToFirstTokenMs": 10571, "firstTurnTtftMs": 8711, "timeToFirstToolCallMs": 19701 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 21540, "peakCpuPercent": 116.3, "firstAtMs": 0, "lastAtMs": 120588 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 76235, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15638, "totalOutputTokens": 5651, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15614, "todayTokensCached": 15016, "todayTokensOut": 240, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15614, "totalTokensCached": 15016, "totalTokensOut": 240, "totalCost": 0, "medianOutputTokensPerSec": 67.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 67.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7647, "cachedInputTokens": 7081, "outputTokens": 48, "cost": 0, "durationMs": 34211, "outputTokensPerSec": 71.51715270433745, "at": "2026-10-28T04:17:25.457Z" }, "lastUpdated": "2026-10-28T04:17:25.457Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:17:26.348Z" } }, "source": "engine-log" }, "derived": { "durationMs": 121124, "meanTokensPerSec": 87.62, "genTokensPerSec": 87.62, "genTokens": 5651, "promptTokensPerSec": 961.99, "peakRssMb": 21540, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15614, "output": 240 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-productivity-plan-and-estimate", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-qwen3-8-27b-q4-2026-10-28T04-17-54-494Z-jb7g", "success": true, "includedInCapability": true, "durationMs": 84576, "timing": { "startedAt": "2026-10-28T04:17:54.495Z", "finishedAt": "2026-10-28T04:19:19.071Z", "timeToFirstArtifactMs": 76765, "timeToLastArtifactWriteMs": 76765, "timeToFirstTokenMs": 15236, "firstTurnTtftMs": 13412, "timeToFirstToolCallMs": 16230 }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 37049, "peakCpuPercent": 95.6, "firstAtMs": 0, "lastAtMs": 80319 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 94491, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4884, "totalOutputTokens": 2071, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:19:16.391Z" } }, "source": "engine-log" }, "derived": { "durationMs": 80843, "meanTokensPerSec": 32.26, "genTokensPerSec": 32.26, "genTokens": 2071, "promptTokensPerSec": 603.51, "peakRssMb": 37049, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-plan-and-estimate", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T04-19-43-086Z-q6yd", "success": true, "includedInCapability": true, "durationMs": 42611, "timing": { "startedAt": "2026-10-28T04:19:43.087Z", "finishedAt": "2026-10-28T04:20:25.698Z", "timeToFirstArtifactMs": 37449, "timeToLastArtifactWriteMs": 39341, "timeToFirstTokenMs": 12844, "firstTurnTtftMs": 11224, "timeToFirstToolCallMs": 23228 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 10432, "peakCpuPercent": 105.7, "firstAtMs": 0, "lastAtMs": 40179 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 75008, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6185, "totalOutputTokens": 734, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6153, "todayTokensCached": 5631, "todayTokensOut": 42, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6153, "totalTokensCached": 5631, "totalTokensOut": 42, "totalCost": 0, "medianOutputTokensPerSec": 38.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6153, "cachedInputTokens": 5631, "outputTokens": 42, "cost": 0, "durationMs": 37720, "outputTokensPerSec": 38.35042655520582, "at": "2026-10-28T04:20:22.427Z" }, "lastUpdated": "2026-10-28T04:20:22.427Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:20:24.870Z" } }, "source": "engine-log" }, "derived": { "durationMs": 40729, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 734, "promptTokensPerSec": null, "peakRssMb": 10432, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 6153, "output": 42 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1075, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-plan-and-estimate", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T04-20-49-322Z-he10", "success": true, "includedInCapability": true, "durationMs": 30007, "timing": { "startedAt": "2026-10-28T04:20:49.322Z", "finishedAt": "2026-10-28T04:21:19.329Z", "timeToFirstArtifactMs": 22634, "timeToLastArtifactWriteMs": 25278, "timeToFirstTokenMs": 10275, "firstTurnTtftMs": 8434, "timeToFirstToolCallMs": 13827 }, "metrics": { "process": { "sampleCount": 6, "peakRssMb": 14917, "peakCpuPercent": 119.5, "firstAtMs": 0, "lastAtMs": 25096 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 69680, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6196, "totalOutputTokens": 946, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6191, "todayTokensCached": 5650, "todayTokensOut": 139, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6191, "totalTokensCached": 5650, "totalTokensOut": 139, "totalCost": 0, "medianOutputTokensPerSec": 74, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 74, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6191, "cachedInputTokens": 5650, "outputTokens": 139, "cost": 0, "durationMs": 23436, "outputTokensPerSec": 74.04688777711243, "at": "2026-10-28T04:21:14.599Z" }, "lastUpdated": "2026-10-28T04:21:14.599Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:21:15.998Z" } }, "source": "engine-log" }, "derived": { "durationMs": 25629, "meanTokensPerSec": 69.48, "genTokensPerSec": 69.48, "genTokens": 946, "promptTokensPerSec": 1013.63, "peakRssMb": 14917, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6191, "output": 139 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-plan-and-estimate", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 1 }, "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T04-21-39-794Z-ozts", "success": true, "includedInCapability": true, "durationMs": 52681, "timing": { "startedAt": "2026-10-28T04:21:39.795Z", "finishedAt": "2026-10-28T04:22:32.476Z", "timeToFirstArtifactMs": 51134, "timeToLastArtifactWriteMs": 51134, "timeToFirstTokenMs": 14260, "firstTurnTtftMs": 12620, "timeToFirstToolCallMs": 15087 }, "metrics": { "process": { "sampleCount": 11, "peakRssMb": 18056, "peakCpuPercent": 109.9, "firstAtMs": 0, "lastAtMs": 50197 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 11, "peakUsedMb": 90578, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4818, "totalOutputTokens": 1126, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:22:31.586Z" } }, "source": "engine-log" }, "derived": { "durationMs": 50722, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 1126, "promptTokensPerSec": null, "peakRssMb": 18056, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 678, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-productivity-meeting-followup", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "meeting-followup", "repetition": 1 }, "trialId": "meeting-followup-mlx-gemma4-12b-q4-2026-10-28T04-22-52-767Z-gfxj", "success": true, "includedInCapability": true, "durationMs": 198865, "timing": { "startedAt": "2026-10-28T04:22:52.767Z", "finishedAt": "2026-10-28T04:26:11.632Z", "timeToFirstArtifactMs": 30833, "timeToLastArtifactWriteMs": 194569, "timeToFirstTokenMs": 12760, "firstTurnTtftMs": 11133, "timeToFirstToolCallMs": 23104 }, "metrics": { "process": { "sampleCount": 40, "peakRssMb": 10551, "peakCpuPercent": 115.9, "firstAtMs": 0, "lastAtMs": 196115 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 74597, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 27623, "totalOutputTokens": 2717, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 18947, "todayTokensCached": 6916, "todayTokensOut": 431, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 18947, "totalTokensCached": 6916, "totalTokensOut": 431, "totalCost": 0, "medianOutputTokensPerSec": 50.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 50.4, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7517, "cachedInputTokens": 6916, "outputTokens": 92, "cost": 0, "durationMs": 2364, "outputTokensPerSec": 50.46249195842284, "at": "2026-10-28T04:26:07.339Z" }, "lastUpdated": "2026-10-28T04:26:07.339Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:26:10.490Z" } }, "source": "engine-log" }, "derived": { "durationMs": 196661, "meanTokensPerSec": 49, "genTokensPerSec": 49, "genTokens": 2717, "promptTokensPerSec": null, "peakRssMb": 10551, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 18947, "output": 431 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1532, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-meeting-followup", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "meeting-followup", "repetition": 1 }, "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T04-26-33-805Z-lxu3", "success": true, "includedInCapability": true, "durationMs": 173790, "timing": { "startedAt": "2026-10-28T04:26:33.805Z", "finishedAt": "2026-10-28T04:29:27.595Z", "timeToFirstArtifactMs": 22005, "timeToLastArtifactWriteMs": 164461, "timeToFirstTokenMs": 10060, "firstTurnTtftMs": 8199, "timeToFirstToolCallMs": 14117 }, "metrics": { "process": { "sampleCount": 34, "peakRssMb": 19936, "peakCpuPercent": 112.6, "firstAtMs": 0, "lastAtMs": 166002 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 75976, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 23416, "totalOutputTokens": 4672, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 32901, "todayTokensCached": 27143, "todayTokensOut": 2794, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 32901, "totalTokensCached": 27143, "totalTokensOut": 2794, "totalCost": 0, "medianOutputTokensPerSec": 102.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 102.8, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10166, "cachedInputTokens": 9495, "outputTokens": 2374, "cost": 0, "durationMs": 22457, "outputTokensPerSec": 109.33577296935871, "at": "2026-10-28T04:29:18.266Z" }, "lastUpdated": "2026-10-28T04:29:18.266Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:29:21.406Z" } }, "source": "engine-log" }, "derived": { "durationMs": 166551, "meanTokensPerSec": 107.2, "genTokensPerSec": 107.2, "genTokens": 4672, "promptTokensPerSec": 1327.01, "peakRssMb": 19936, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32901, "output": 2794 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-meeting-followup", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "meeting-followup", "repetition": 1 }, "trialId": "meeting-followup-mlx-qwen3-8-27b-q4-2026-10-28T04-29-49-275Z-4vcj", "success": true, "includedInCapability": true, "durationMs": 138699, "timing": { "startedAt": "2026-10-28T04:29:49.276Z", "finishedAt": "2026-10-28T04:32:07.975Z", "timeToFirstArtifactMs": 37290, "timeToLastArtifactWriteMs": 132587, "timeToFirstTokenMs": 17014, "firstTurnTtftMs": 15374, "timeToFirstToolCallMs": 21652 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 17969, "peakCpuPercent": 123.9, "firstAtMs": 0, "lastAtMs": 135839 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 93261, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12684, "totalOutputTokens": 490, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 2885, "todayTokensCached": 0, "todayTokensOut": 545, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 2885, "totalTokensCached": 0, "totalTokensOut": 545, "totalCost": 0, "medianOutputTokensPerSec": 47.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 47.5, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 2885, "cachedInputTokens": 0, "outputTokens": 545, "cost": 0, "durationMs": 35652, "outputTokensPerSec": 47.470778998646985, "at": "2026-10-28T04:30:26.568Z" }, "lastUpdated": "2026-10-28T04:30:26.568Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:32:06.710Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136366, "meanTokensPerSec": 48, "genTokensPerSec": 48, "genTokens": 490, "promptTokensPerSec": null, "peakRssMb": 17969, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 2885, "output": 545 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 862.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-meeting-followup", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "meeting-followup", "repetition": 1 }, "trialId": "meeting-followup-qwen3-8-27b-q4-2026-10-28T04-32-30-073Z-nrkc", "success": true, "includedInCapability": true, "durationMs": 145747, "timing": { "startedAt": "2026-10-28T04:32:30.074Z", "finishedAt": "2026-10-28T04:34:55.821Z", "timeToFirstArtifactMs": 40227, "timeToLastArtifactWriteMs": 138059, "timeToFirstTokenMs": 20045, "firstTurnTtftMs": 18233, "timeToFirstToolCallMs": 24446 }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 38586, "peakCpuPercent": 118.9, "firstAtMs": 1, "lastAtMs": 140856 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 96148, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12359, "totalOutputTokens": 1135, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 2913, "todayTokensCached": 0, "todayTokensOut": 470, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 2913, "totalTokensCached": 0, "totalTokensOut": 470, "totalCost": 0, "medianOutputTokensPerSec": 44.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 44.7, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 2913, "outputTokens": 470, "cost": 0, "durationMs": 38416, "outputTokensPerSec": 44.700818091688724, "at": "2026-10-28T04:33:10.302Z" }, "lastUpdated": "2026-10-28T04:33:10.302Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:34:52.506Z" } }, "source": "engine-log" }, "derived": { "durationMs": 141382, "meanTokensPerSec": 50.07, "genTokensPerSec": 50.07, "genTokens": 1135, "promptTokensPerSec": 615.57, "peakRssMb": 38586, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 2913, "output": 470 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-productivity-craftbook-spreadsheet-model", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 1 }, "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T04-35-16-293Z-uiv3", "success": true, "includedInCapability": true, "durationMs": 171317, "timing": { "startedAt": "2026-10-28T04:35:16.293Z", "finishedAt": "2026-10-28T04:38:07.610Z", "timeToFirstArtifactMs": 24715, "timeToLastArtifactWriteMs": 120920, "timeToFirstTokenMs": 15679, "firstTurnTtftMs": 9018, "timeToFirstToolCallMs": 24714 }, "metrics": { "process": { "sampleCount": 34, "peakRssMb": 26586, "peakCpuPercent": 112.4, "firstAtMs": 0, "lastAtMs": 165679 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 85685, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 54158, "totalOutputTokens": 6858, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 48533, "todayTokensCached": 35532, "todayTokensOut": 1392, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 48533, "totalTokensCached": 35532, "totalTokensOut": 1392, "totalCost": 0, "medianOutputTokensPerSec": 92.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 92.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7891, "cachedInputTokens": 7685, "outputTokens": 130, "cost": 0, "durationMs": 31781, "outputTokensPerSec": 104.99217442048796, "at": "2026-10-28T04:37:49.712Z" }, "lastUpdated": "2026-10-28T04:37:49.712Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:38:03.549Z" } }, "source": "engine-log" }, "derived": { "durationMs": 166202, "meanTokensPerSec": 99.39, "genTokensPerSec": 99.39, "genTokens": 6858, "promptTokensPerSec": 1269.45, "peakRssMb": 26586, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 48533, "output": 1392 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-craftbook-spreadsheet-model", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 1 }, "trialId": "craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-28T04-38-32-385Z-nyt0", "success": true, "includedInCapability": true, "durationMs": 1235584, "timing": { "startedAt": "2026-10-28T04:38:32.385Z", "finishedAt": "2026-10-28T04:59:07.969Z", "timeToFirstArtifactMs": 74924, "timeToLastArtifactWriteMs": 1218530, "timeToFirstTokenMs": 25888, "firstTurnTtftMs": 19273, "timeToFirstToolCallMs": 27428 }, "metrics": { "process": { "sampleCount": 244, "peakRssMb": 18591, "peakCpuPercent": 118.8, "firstAtMs": 1, "lastAtMs": 1219971 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 244, "peakUsedMb": 90343, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 54222, "totalOutputTokens": 27838, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 54046, "todayTokensCached": 41500, "todayTokensOut": 7726, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 54046, "totalTokensCached": 41500, "totalTokensOut": 7726, "totalCost": 0, "medianOutputTokensPerSec": 26.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.5, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16317, "cachedInputTokens": 15814, "outputTokens": 1027, "cost": 0, "durationMs": 396516, "outputTokensPerSec": 23.154080583122084, "at": "2026-10-28T04:58:50.915Z" }, "lastUpdated": "2026-10-28T04:58:50.915Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T04:58:54.068Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1220514, "meanTokensPerSec": 27, "genTokensPerSec": 27, "genTokens": 27838, "promptTokensPerSec": null, "peakRssMb": 18591, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 54046, "output": 7726 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 611, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-craftbook-spreadsheet-model", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 1 }, "trialId": "craftbook-spreadsheet-model-qwen3-8-27b-q4-2026-10-28T04-59-34-526Z-rmt6", "success": true, "includedInCapability": true, "durationMs": 617803, "timing": { "startedAt": "2026-10-28T04:59:34.527Z", "finishedAt": "2026-10-28T05:09:52.330Z", "timeToFirstArtifactMs": 111970, "timeToLastArtifactWriteMs": 598543, "timeToFirstTokenMs": 32821, "firstTurnTtftMs": 26068, "timeToFirstToolCallMs": 38704 }, "metrics": { "process": { "sampleCount": 121, "peakRssMb": 42469, "peakCpuPercent": 118.3, "firstAtMs": 0, "lastAtMs": 602264 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 121, "peakUsedMb": 103292, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 35680, "totalOutputTokens": 12455, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35648, "todayTokensCached": 33318, "todayTokensOut": 2860, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35648, "totalTokensCached": 33318, "totalTokensOut": 2860, "totalCost": 0, "medianOutputTokensPerSec": 27, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16173, "cachedInputTokens": 15445, "outputTokens": 888, "cost": 0, "durationMs": 241906, "outputTokensPerSec": 24.113920795203704, "at": "2026-10-28T05:09:33.070Z" }, "lastUpdated": "2026-10-28T05:09:33.070Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T05:09:38.351Z" } }, "source": "engine-log" }, "derived": { "durationMs": 602788, "meanTokensPerSec": 26.16, "genTokensPerSec": 26.16, "genTokens": 12455, "promptTokensPerSec": 382.42, "peakRssMb": 42469, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35648, "output": 2860 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-craftbook-spreadsheet-model", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 1 }, "trialId": "craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-28T05-10-15-645Z-m954", "success": true, "includedInCapability": true, "durationMs": 825385, "timing": { "startedAt": "2026-10-28T05:10:15.646Z", "finishedAt": "2026-10-28T05:24:01.031Z", "timeToFirstArtifactMs": 46604, "timeToLastArtifactWriteMs": 778357, "timeToFirstTokenMs": 21217, "firstTurnTtftMs": 14674, "timeToFirstToolCallMs": 30904 }, "metrics": { "process": { "sampleCount": 163, "peakRssMb": 13315, "peakCpuPercent": 131.5, "firstAtMs": 0, "lastAtMs": 815136 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 163, "peakUsedMb": 75608, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 97500, "totalOutputTokens": 20100, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 96511, "todayTokensCached": 89847, "todayTokensOut": 2949, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 96511, "totalTokensCached": 89847, "totalTokensOut": 2949, "totalCost": 0, "medianOutputTokensPerSec": 35.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.7, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8479, "cachedInputTokens": 8254, "outputTokens": 39, "cost": 0, "durationMs": 24293, "outputTokensPerSec": 41.378863505336334, "at": "2026-10-28T05:23:41.666Z" }, "lastUpdated": "2026-10-28T05:23:41.666Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T05:23:52.616Z" } }, "source": "engine-log" }, "derived": { "durationMs": 815914, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 20100, "promptTokensPerSec": null, "peakRssMb": 13315, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 96511, "output": 2949 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1063, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-productivity-conflict-synthesis", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T05-24-23-449Z-rp3g", "success": false, "includedInCapability": true, "durationMs": 1469753, "timing": { "startedAt": "2026-10-28T05:24:23.449Z", "finishedAt": "2026-10-28T05:48:53.202Z", "timeToFirstArtifactMs": 204732, "timeToLastArtifactWriteMs": 1462496, "timeToFirstTokenMs": 28884, "firstTurnTtftMs": 27237, "timeToFirstToolCallMs": 34169 }, "metrics": { "process": { "sampleCount": 293, "peakRssMb": 18023, "peakCpuPercent": 118.8, "firstAtMs": 0, "lastAtMs": 1465965 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 293, "peakUsedMb": 91800, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 309801, "totalOutputTokens": 19160, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 134020, "todayTokensCached": 117302, "todayTokensOut": 6959, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 134020, "totalTokensCached": 117302, "totalTokensOut": 6959, "totalCost": 0, "medianOutputTokensPerSec": 29.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.4, "turns": 11 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 18889, "cachedInputTokens": 18383, "outputTokens": 733, "cost": 0, "durationMs": 176114, "outputTokensPerSec": 26.227550508207546, "at": "2026-10-28T05:48:45.945Z" }, "lastUpdated": "2026-10-28T05:48:45.945Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T05:48:51.031Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1466520, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 19160, "promptTokensPerSec": null, "peakRssMb": 18023, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 134020, "output": 6959 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 519, "sampleCount": 11, "structuredSampleCount": 0, "firstTokenSampleCount": 11, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-conflict-synthesis", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-qwen3-8-27b-q4-2026-10-28T05-49-14-782Z-es79", "success": true, "includedInCapability": true, "durationMs": 280328, "timing": { "startedAt": "2026-10-28T05:49:14.783Z", "finishedAt": "2026-10-28T05:53:55.111Z", "timeToFirstArtifactMs": 96656, "timeToLastArtifactWriteMs": 277154, "timeToFirstTokenMs": 34187, "firstTurnTtftMs": 32361, "timeToFirstToolCallMs": 36966 }, "metrics": { "process": { "sampleCount": 56, "peakRssMb": 44778, "peakCpuPercent": 113.5, "firstAtMs": 0, "lastAtMs": 276115 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 56, "peakUsedMb": 100943, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 53989, "totalOutputTokens": 4238, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11966, "todayTokensCached": 11896, "todayTokensOut": 613, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11966, "totalTokensCached": 11896, "totalTokensOut": 613, "totalCost": 0, "medianOutputTokensPerSec": 26.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.5, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11966, "cachedInputTokens": 11896, "outputTokens": 613, "cost": 0, "durationMs": 118439, "outputTokensPerSec": 26.481798328730246, "at": "2026-10-28T05:51:15.048Z" }, "lastUpdated": "2026-10-28T05:51:15.048Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T05:53:52.474Z" } }, "source": "engine-log" }, "derived": { "durationMs": 276642, "meanTokensPerSec": 30.6, "genTokensPerSec": 30.6, "genTokens": 4238, "promptTokensPerSec": 423.65, "peakRssMb": 44778, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11966, "output": 613 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-conflict-synthesis", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T05-54-16-421Z-j9gb", "success": true, "includedInCapability": true, "durationMs": 163883, "timing": { "startedAt": "2026-10-28T05:54:16.421Z", "finishedAt": "2026-10-28T05:57:00.304Z", "timeToFirstArtifactMs": 59595, "timeToLastArtifactWriteMs": 161137, "timeToFirstTokenMs": 17593, "firstTurnTtftMs": 15960, "timeToFirstToolCallMs": 29114 }, "metrics": { "process": { "sampleCount": 33, "peakRssMb": 10609, "peakCpuPercent": 105.5, "firstAtMs": 0, "lastAtMs": 160770 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 76009, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 46854, "totalOutputTokens": 2143, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 30845, "todayTokensCached": 18921, "todayTokensOut": 1067, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 30845, "totalTokensCached": 18921, "totalTokensOut": 1067, "totalCost": 0, "medianOutputTokensPerSec": 35.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.6, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10828, "cachedInputTokens": 0, "outputTokens": 514, "cost": 0, "durationMs": 24364, "outputTokensPerSec": 35.57637977009889, "at": "2026-10-28T05:56:57.558Z" }, "lastUpdated": "2026-10-28T05:56:57.558Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T05:56:58.805Z" } }, "source": "engine-log" }, "derived": { "durationMs": 161318, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 2143, "promptTokensPerSec": null, "peakRssMb": 10609, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 30845, "output": 1067 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1082.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-conflict-synthesis", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 1 }, "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T05-57-26-010Z-sg4v", "success": true, "includedInCapability": true, "durationMs": 126086, "timing": { "startedAt": "2026-10-28T05:57:26.011Z", "finishedAt": "2026-10-28T05:59:32.097Z", "timeToFirstArtifactMs": 26812, "timeToLastArtifactWriteMs": 121272, "timeToFirstTokenMs": 15430, "firstTurnTtftMs": 13592, "timeToFirstToolCallMs": 19493 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 20436, "peakCpuPercent": 119.3, "firstAtMs": 0, "lastAtMs": 120480 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 75162, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 86378, "totalOutputTokens": 2036, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 25787, "todayTokensCached": 17172, "todayTokensOut": 1118, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 25787, "totalTokensCached": 17172, "totalTokensOut": 1118, "totalCost": 0, "medianOutputTokensPerSec": 77.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 77.8, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8171, "cachedInputTokens": 6517, "outputTokens": 534, "cost": 0, "durationMs": 44838, "outputTokensPerSec": 78.0321263974046, "at": "2026-10-28T05:59:27.283Z" }, "lastUpdated": "2026-10-28T05:59:27.283Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T05:59:28.086Z" } }, "source": "engine-log" }, "derived": { "durationMs": 121028, "meanTokensPerSec": 75.39, "genTokensPerSec": 75.39, "genTokens": 2036, "promptTokensPerSec": 1060.14, "peakRssMb": 20436, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 25787, "output": 1118 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-productivity-docblocks-theme-roundtrip", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 1 }, "trialId": "docblocks-theme-roundtrip-qwen3-8-27b-q4-2026-10-28T05-59-57-731Z-trgj", "success": false, "includedInCapability": true, "durationMs": 311573, "timing": { "startedAt": "2026-10-28T05:59:57.732Z", "finishedAt": "2026-10-28T06:05:09.305Z", "timeToFirstArtifactMs": 116622, "timeToLastArtifactWriteMs": 302117, "timeToFirstTokenMs": 20286, "firstTurnTtftMs": 18454, "timeToFirstToolCallMs": 20727 }, "metrics": { "process": { "sampleCount": 61, "peakRssMb": 42931, "peakCpuPercent": 119.4, "firstAtMs": 0, "lastAtMs": 301461 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 61, "peakUsedMb": 98874, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 30084, "totalOutputTokens": 6829, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 81613, "todayTokensCached": 76137, "todayTokensOut": 5623, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 81613, "totalTokensCached": 76137, "totalTokensOut": 5623, "totalCost": 0, "medianOutputTokensPerSec": 43.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 43.7, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 15317, "cachedInputTokens": 14274, "outputTokens": 927, "cost": 0, "durationMs": 34688, "outputTokensPerSec": 34.29835927909145, "at": "2026-10-28T06:04:59.849Z" }, "lastUpdated": "2026-10-28T06:04:59.849Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T06:05:00.789Z" } }, "source": "engine-log" }, "derived": { "durationMs": 302010, "meanTokensPerSec": 42.76, "genTokensPerSec": 42.76, "genTokens": 6829, "promptTokensPerSec": 482.6, "peakRssMb": 42931, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 81613, "output": 5623 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-docblocks-theme-roundtrip", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 1 }, "trialId": "docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-28T06-05-32-061Z-39pc", "success": false, "includedInCapability": true, "durationMs": 374445, "timing": { "startedAt": "2026-10-28T06:05:32.062Z", "finishedAt": "2026-10-28T06:11:46.507Z", "timeToFirstArtifactMs": 128100, "timeToLastArtifactWriteMs": 370366, "timeToFirstTokenMs": 13604, "firstTurnTtftMs": 11978, "timeToFirstToolCallMs": 128102 }, "metrics": { "process": { "sampleCount": 75, "peakRssMb": 10462, "peakCpuPercent": 114.6, "firstAtMs": 0, "lastAtMs": 371784 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 75, "peakUsedMb": 74910, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 78688, "totalOutputTokens": 7245, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 63750, "todayTokensCached": 17208, "todayTokensOut": 3275, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 63750, "totalTokensCached": 17208, "totalTokensOut": 3275, "totalCost": 0, "medianOutputTokensPerSec": 35.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.4, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10806, "cachedInputTokens": 525, "outputTokens": 123, "cost": 0, "durationMs": 12527, "outputTokensPerSec": 35.533333731565946, "at": "2026-10-28T06:11:42.428Z" }, "lastUpdated": "2026-10-28T06:11:42.428Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T06:11:45.454Z" } }, "source": "engine-log" }, "derived": { "durationMs": 372331, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 7245, "promptTokensPerSec": null, "peakRssMb": 10462, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 63750, "output": 3275 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1085, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-docblocks-theme-roundtrip", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 1 }, "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T06-12-08-690Z-rb23", "success": false, "includedInCapability": true, "durationMs": 333317, "timing": { "startedAt": "2026-10-28T06:12:08.691Z", "finishedAt": "2026-10-28T06:17:42.008Z", "timeToFirstArtifactMs": 111138, "timeToLastArtifactWriteMs": 319202, "timeToFirstTokenMs": 11091, "firstTurnTtftMs": 9234, "timeToFirstToolCallMs": 103325 }, "metrics": { "process": { "sampleCount": 65, "peakRssMb": 24320, "peakCpuPercent": 106.1, "firstAtMs": 0, "lastAtMs": 321605 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 65, "peakUsedMb": 79076, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 94139, "totalOutputTokens": 11538, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 82818, "todayTokensCached": 47123, "todayTokensOut": 2442, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 82818, "totalTokensCached": 47123, "totalTokensOut": 2442, "totalCost": 0, "medianOutputTokensPerSec": 85.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 85.4, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12516, "cachedInputTokens": 12483, "outputTokens": 147, "cost": 0, "durationMs": 36656, "outputTokensPerSec": 60.00550735478461, "at": "2026-10-28T06:17:27.893Z" }, "lastUpdated": "2026-10-28T06:17:27.893Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T06:17:31.892Z" } }, "source": "engine-log" }, "derived": { "durationMs": 322153, "meanTokensPerSec": 74.4, "genTokensPerSec": 74.4, "genTokens": 11538, "promptTokensPerSec": 1121.63, "peakRssMb": 24320, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 82818, "output": 2442 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-docblocks-theme-roundtrip", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 1 }, "trialId": "docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-28T06-18-05-860Z-y8gy", "success": false, "includedInCapability": true, "durationMs": 364792, "timing": { "startedAt": "2026-10-28T06:18:05.860Z", "finishedAt": "2026-10-28T06:24:10.652Z", "timeToFirstArtifactMs": 68227, "timeToLastArtifactWriteMs": 358476, "timeToFirstTokenMs": 19359, "firstTurnTtftMs": 17709, "timeToFirstToolCallMs": 20831 }, "metrics": { "process": { "sampleCount": 73, "peakRssMb": 18022, "peakCpuPercent": 126.7, "firstAtMs": 1, "lastAtMs": 361545 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 73, "peakUsedMb": 90787, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 111125, "totalOutputTokens": 2165, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 74532, "todayTokensCached": 18796, "todayTokensOut": 2165, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 74532, "totalTokensCached": 18796, "totalTokensOut": 2165, "totalCost": 0, "medianOutputTokensPerSec": 29.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.5, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13468, "cachedInputTokens": 1221, "outputTokens": 208, "cost": 0, "durationMs": 56043, "outputTokensPerSec": 27.096151428637747, "at": "2026-10-28T06:24:04.338Z" }, "lastUpdated": "2026-10-28T06:24:04.338Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T06:24:09.025Z" } }, "source": "engine-log" }, "derived": { "durationMs": 362097, "meanTokensPerSec": 30, "genTokensPerSec": 30, "genTokens": 2165, "promptTokensPerSec": null, "peakRssMb": 18022, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 74532, "output": 2165 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 511, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-productivity-craftbook-powerpoint-deck", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 1 }, "trialId": "craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-28T06-24-33-379Z-5bn5", "success": false, "includedInCapability": true, "durationMs": 4211333, "timing": { "startedAt": "2026-10-28T06:24:33.380Z", "finishedAt": "2026-10-28T07:34:44.713Z", "timeToFirstArtifactMs": 50167, "timeToLastArtifactWriteMs": 3670600, "timeToFirstTokenMs": 21991, "firstTurnTtftMs": 15430, "timeToFirstToolCallMs": 28088 }, "metrics": { "process": { "sampleCount": 836, "peakRssMb": 19215, "peakCpuPercent": 162.4, "firstAtMs": 1, "lastAtMs": 4196398 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 836, "peakUsedMb": 102219, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 489224, "totalOutputTokens": 83697, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 46, "todayTokensIn": 749893, "todayTokensCached": 667026, "todayTokensOut": 44842, "todayCost": 0, "totalTurns": 46, "totalTokensIn": 749893, "totalTokensCached": 667026, "totalTokensOut": 44842, "totalCost": 0, "medianOutputTokensPerSec": 37.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.2, "turns": 46 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22331, "cachedInputTokens": 22201, "outputTokens": 8192, "cost": 0, "durationMs": 299474, "outputTokensPerSec": 34.152151616097775, "at": "2026-10-28T07:34:08.772Z" }, "lastUpdated": "2026-10-28T07:34:08.772Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:34:35.657Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4201213, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 83697, "promptTokensPerSec": null, "peakRssMb": 19215, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 749893, "output": 44842 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 929, "sampleCount": 50, "structuredSampleCount": 0, "firstTokenSampleCount": 50, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-craftbook-powerpoint-deck", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 1 }, "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T07-35-06-880Z-zkew", "success": true, "includedInCapability": true, "durationMs": 234113, "timing": { "startedAt": "2026-10-28T07:35:06.880Z", "finishedAt": "2026-10-28T07:39:00.993Z", "timeToFirstArtifactMs": 37568, "timeToLastArtifactWriteMs": 224244, "timeToFirstTokenMs": 20205, "firstTurnTtftMs": 13462, "timeToFirstToolCallMs": 26087 }, "metrics": { "process": { "sampleCount": 46, "peakRssMb": 27064, "peakCpuPercent": 122.6, "firstAtMs": 0, "lastAtMs": 225862 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 95042, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 74040, "totalOutputTokens": 6463, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43025, "todayTokensCached": 41338, "todayTokensOut": 1046, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43025, "totalTokensCached": 41338, "totalTokensOut": 1046, "totalCost": 0, "medianOutputTokensPerSec": 91.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 91.3, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11958, "cachedInputTokens": 11220, "outputTokens": 39, "cost": 0, "durationMs": 57551, "outputTokensPerSec": 88.68144690781797, "at": "2026-10-28T07:37:56.000Z" }, "lastUpdated": "2026-10-28T07:37:56.000Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:38:54.318Z" } }, "source": "engine-log" }, "derived": { "durationMs": 226388, "meanTokensPerSec": 79.71, "genTokensPerSec": 79.71, "genTokens": 6463, "promptTokensPerSec": 1026.23, "peakRssMb": 27064, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43025, "output": 1046 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-craftbook-powerpoint-deck", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 1 }, "trialId": "craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-28T07-39-23-343Z-881f", "success": true, "includedInCapability": true, "durationMs": 1396902, "timing": { "startedAt": "2026-10-28T07:39:23.344Z", "finishedAt": "2026-10-28T08:02:40.246Z", "timeToFirstArtifactMs": 532549, "timeToLastArtifactWriteMs": 1390548, "timeToFirstTokenMs": 29841, "firstTurnTtftMs": 23269, "timeToFirstToolCallMs": 31790 }, "metrics": { "process": { "sampleCount": 278, "peakRssMb": 20582, "peakCpuPercent": 112.8, "firstAtMs": 0, "lastAtMs": 1390562 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 278, "peakUsedMb": 99597, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 112470, "totalOutputTokens": 23027, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 61563, "todayTokensCached": 56385, "todayTokensOut": 5485, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 61563, "totalTokensCached": 56385, "totalTokensOut": 5485, "totalCost": 0, "medianOutputTokensPerSec": 27.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27.1, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13325, "cachedInputTokens": 13250, "outputTokens": 86, "cost": 0, "durationMs": 344563, "outputTokensPerSec": 26.54013924529026, "at": "2026-10-28T07:57:04.720Z" }, "lastUpdated": "2026-10-28T07:57:04.720Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:02:36.509Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1392097, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 23027, "promptTokensPerSec": null, "peakRssMb": 20582, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 61563, "output": 5485 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 524.5, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-craftbook-powerpoint-deck", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 1 }, "trialId": "craftbook-powerpoint-deck-qwen3-8-27b-q4-2026-10-28T08-03-03-332Z-ap9o", "success": true, "includedInCapability": true, "durationMs": 453693, "timing": { "startedAt": "2026-10-28T08:03:03.332Z", "finishedAt": "2026-10-28T08:10:37.025Z", "timeToFirstArtifactMs": 70488, "timeToLastArtifactWriteMs": 442605, "timeToFirstTokenMs": 36475, "firstTurnTtftMs": 29776, "timeToFirstToolCallMs": 41742 }, "metrics": { "process": { "sampleCount": 89, "peakRssMb": 46710, "peakCpuPercent": 145.8, "firstAtMs": 0, "lastAtMs": 441750 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 89, "peakUsedMb": 108984, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 82502, "totalOutputTokens": 5139, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 41442, "todayTokensCached": 39296, "todayTokensOut": 2109, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 41442, "totalTokensCached": 39296, "totalTokensOut": 2109, "totalCost": 0, "medianOutputTokensPerSec": 35.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 35.2, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11418, "cachedInputTokens": 10433, "outputTokens": 73, "cost": 0, "durationMs": 90331, "outputTokensPerSec": 38.44568372444697, "at": "2026-10-28T08:07:35.205Z" }, "lastUpdated": "2026-10-28T08:07:35.205Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:10:26.910Z" } }, "source": "engine-log" }, "derived": { "durationMs": 442529, "meanTokensPerSec": 31.24, "genTokensPerSec": 31.24, "genTokens": 5139, "promptTokensPerSec": 359.98, "peakRssMb": 46710, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 41442, "output": 2109 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-productivity-wikipedia-research-brief", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 1 }, "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T08-11-04-950Z-fyq5", "success": true, "includedInCapability": true, "durationMs": 246035, "timing": { "startedAt": "2026-10-28T08:11:04.950Z", "finishedAt": "2026-10-28T08:15:10.985Z", "timeToFirstArtifactMs": 46188, "timeToLastArtifactWriteMs": 239553, "timeToFirstTokenMs": 10044, "firstTurnTtftMs": 8147, "timeToFirstToolCallMs": 27440 }, "metrics": { "process": { "sampleCount": 49, "peakRssMb": 25591, "peakCpuPercent": 118.6, "firstAtMs": 0, "lastAtMs": 241059 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 82246, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 50030, "totalOutputTokens": 11331, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 83654, "todayTokensCached": 59555, "todayTokensOut": 7503, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 83654, "totalTokensCached": 59555, "totalTokensOut": 7503, "totalCost": 0, "medianOutputTokensPerSec": 68.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 68.6, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17771, "outputTokens": 3134, "cost": 0, "durationMs": 67646, "outputTokensPerSec": 63.17640137780613, "at": "2026-10-28T08:15:04.503Z" }, "lastUpdated": "2026-10-28T08:15:04.503Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:15:07.625Z" } }, "source": "engine-log" }, "derived": { "durationMs": 241605, "meanTokensPerSec": 67.22, "genTokensPerSec": 67.22, "genTokens": 11331, "promptTokensPerSec": 980.15, "peakRssMb": 25591, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 83654, "output": 7503 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-wikipedia-research-brief", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 1 }, "trialId": "wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-28T08-15-32-657Z-xw1n", "success": false, "includedInCapability": true, "durationMs": 319399, "timing": { "startedAt": "2026-10-28T08:15:32.658Z", "finishedAt": "2026-10-28T08:20:52.057Z", "timeToFirstArtifactMs": 113473, "timeToLastArtifactWriteMs": 313695, "timeToFirstTokenMs": 13454, "firstTurnTtftMs": 11754, "timeToFirstToolCallMs": 64105 }, "metrics": { "process": { "sampleCount": 64, "peakRssMb": 18012, "peakCpuPercent": 124.9, "firstAtMs": 0, "lastAtMs": 316305 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 64, "peakUsedMb": 92722, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 52705, "totalOutputTokens": 5898, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 28906, "todayTokensCached": 5764, "todayTokensOut": 4541, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 28906, "totalTokensCached": 5764, "totalTokensOut": 4541, "totalCost": 0, "medianOutputTokensPerSec": 34.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 34.3, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 10790, "cachedInputTokens": 1302, "outputTokens": 1597, "cost": 0, "durationMs": 111699, "outputTokensPerSec": 31.004087730191916, "at": "2026-10-28T08:20:46.353Z" }, "lastUpdated": "2026-10-28T08:20:46.353Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:20:50.600Z" } }, "source": "engine-log" }, "derived": { "durationMs": 316845, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 5898, "promptTokensPerSec": null, "peakRssMb": 18012, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 28906, "output": 4541 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 566.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-wikipedia-research-brief", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 1 }, "trialId": "wikipedia-research-brief-qwen3-8-27b-q4-2026-10-28T08-21-21-607Z-lqdt", "success": true, "includedInCapability": true, "durationMs": 603109, "timing": { "startedAt": "2026-10-28T08:21:21.607Z", "finishedAt": "2026-10-28T08:31:24.716Z", "timeToFirstArtifactMs": 344792, "timeToLastArtifactWriteMs": 596659, "timeToFirstTokenMs": 16672, "firstTurnTtftMs": 14782, "timeToFirstToolCallMs": 124905 }, "metrics": { "process": { "sampleCount": 120, "peakRssMb": 44636, "peakCpuPercent": 117.8, "firstAtMs": 1, "lastAtMs": 597628 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 120, "peakUsedMb": 102945, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 58818, "totalOutputTokens": 12814, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 19158, "todayTokensCached": 9427, "todayTokensOut": 1986, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 19158, "totalTokensCached": 9427, "totalTokensOut": 1986, "totalCost": 0, "medianOutputTokensPerSec": 31.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.9, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9658, "outputTokens": 1787, "cost": 0, "durationMs": 112556, "outputTokensPerSec": 36.148068316853646, "at": "2026-10-28T08:29:06.759Z" }, "lastUpdated": "2026-10-28T08:29:06.759Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:31:20.844Z" } }, "source": "engine-log" }, "derived": { "durationMs": 598156, "meanTokensPerSec": 29.77, "genTokensPerSec": 29.77, "genTokens": 12814, "promptTokensPerSec": 375.35, "peakRssMb": 44636, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 19158, "output": 1986 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-wikipedia-research-brief", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 1 }, "trialId": "wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-28T08-31-49-358Z-pkqi", "success": false, "includedInCapability": true, "durationMs": 1803837, "timing": { "startedAt": "2026-10-28T08:31:49.359Z", "finishedAt": "2026-10-28T09:01:53.196Z", "timeToFirstArtifactMs": 56148, "timeToLastArtifactWriteMs": 1800852, "timeToFirstTokenMs": 11921, "firstTurnTtftMs": 10145, "timeToFirstToolCallMs": 33821 }, "metrics": { "process": { "sampleCount": 359, "peakRssMb": 11693, "peakCpuPercent": 111.1, "firstAtMs": 0, "lastAtMs": 1799709 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 359, "peakUsedMb": 77029, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 378353, "totalOutputTokens": 29086, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 430662, "todayTokensCached": 324260, "todayTokensOut": 22902, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 430662, "totalTokensCached": 324260, "totalTokensOut": 22902, "totalCost": 0, "medianOutputTokensPerSec": 34.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.4, "turns": 16 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 46516, "cachedInputTokens": 44629, "outputTokens": 1420, "cost": 0, "durationMs": 45111, "outputTokensPerSec": 33.28720126258862, "at": "2026-10-28T09:01:50.211Z" }, "lastUpdated": "2026-10-28T09:01:50.211Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:01:50.782Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1800248, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 29086, "promptTokensPerSec": null, "peakRssMb": 11693, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 430662, "output": 22902 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1010, "sampleCount": 16, "structuredSampleCount": 0, "firstTokenSampleCount": 16, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-productivity-craftbook-week-plan", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 1 }, "trialId": "craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-28T09-02-14-313Z-9xn3", "success": false, "includedInCapability": true, "durationMs": 5416281, "timing": { "startedAt": "2026-10-28T09:02:14.313Z", "finishedAt": "2026-10-28T10:32:30.594Z", "timeToFirstArtifactMs": 1981774, "timeToLastArtifactWriteMs": 5252581, "timeToFirstTokenMs": 39199, "firstTurnTtftMs": 32621, "timeToFirstToolCallMs": 47753 }, "metrics": { "process": { "sampleCount": 1076, "peakRssMb": 20119, "peakCpuPercent": 119.8, "firstAtMs": 0, "lastAtMs": 5402486 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1076, "peakUsedMb": 94102, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 178448, "totalOutputTokens": 83793, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 185172, "todayTokensCached": 174372, "todayTokensOut": 15144, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 185172, "totalTokensCached": 174372, "totalTokensOut": 15144, "totalCost": 0, "medianOutputTokensPerSec": 24.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.5, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 17843, "cachedInputTokens": 17573, "outputTokens": 408, "cost": 0, "durationMs": 50867, "outputTokensPerSec": 31.075997636687855, "at": "2026-10-28T10:31:38.775Z" }, "lastUpdated": "2026-10-28T10:31:38.775Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T10:32:18.441Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5403045, "meanTokensPerSec": 14, "genTokensPerSec": 14, "genTokens": 83793, "promptTokensPerSec": null, "peakRssMb": 20119, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 185172, "output": 15144 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 565, "sampleCount": 11, "structuredSampleCount": 0, "firstTokenSampleCount": 11, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-craftbook-week-plan", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 1 }, "trialId": "craftbook-week-plan-qwen3-8-27b-q4-2026-10-28T10-32-52-650Z-w19d", "success": true, "includedInCapability": true, "durationMs": 1791397, "timing": { "startedAt": "2026-10-28T10:32:52.650Z", "finishedAt": "2026-10-28T11:02:44.047Z", "timeToFirstArtifactMs": 508299, "timeToLastArtifactWriteMs": 1730635, "timeToFirstTokenMs": 37994, "firstTurnTtftMs": 31293, "timeToFirstToolCallMs": 41383 }, "metrics": { "process": { "sampleCount": 356, "peakRssMb": 49898, "peakCpuPercent": 117.5, "firstAtMs": 0, "lastAtMs": 1782509 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 356, "peakUsedMb": 113211, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 166955, "totalOutputTokens": 30265, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 105363, "todayTokensCached": 87010, "todayTokensOut": 15781, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 105363, "totalTokensCached": 87010, "totalTokensOut": 15781, "totalCost": 0, "medianOutputTokensPerSec": 38.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 38.3, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5710, "cachedInputTokens": 2463, "outputTokens": 1157, "cost": 0, "durationMs": 42392, "outputTokensPerSec": 40.58936317115004, "at": "2026-10-28T11:01:43.285Z" }, "lastUpdated": "2026-10-28T11:01:43.285Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:02:36.764Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1783036, "meanTokensPerSec": 25.01, "genTokensPerSec": 25.01, "genTokens": 30265, "promptTokensPerSec": 428.1, "peakRssMb": 49898, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 105363, "output": 15781 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-craftbook-week-plan", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 1 }, "trialId": "craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-28T11-03-04-929Z-q9pi", "success": false, "includedInCapability": true, "durationMs": 868245, "timing": { "startedAt": "2026-10-28T11:03:04.929Z", "finishedAt": "2026-10-28T11:17:33.174Z", "timeToFirstArtifactMs": 51394, "timeToLastArtifactWriteMs": 859195, "timeToFirstTokenMs": 25728, "firstTurnTtftMs": 19175, "timeToFirstToolCallMs": 35006 }, "metrics": { "process": { "sampleCount": 172, "peakRssMb": 12816, "peakCpuPercent": 124.1, "firstAtMs": 1, "lastAtMs": 858881 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 172, "peakUsedMb": 78397, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 164270, "totalOutputTokens": 20498, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 126180, "todayTokensCached": 78455, "todayTokensOut": 4819, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 126180, "totalTokensCached": 78455, "totalTokensOut": 4819, "totalCost": 0, "medianOutputTokensPerSec": 36.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.2, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10945, "cachedInputTokens": 1689, "outputTokens": 70, "cost": 0, "durationMs": 19206, "outputTokensPerSec": 44.529968051707826, "at": "2026-10-28T11:17:24.126Z" }, "lastUpdated": "2026-10-28T11:17:24.126Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:17:25.444Z" } }, "source": "engine-log" }, "derived": { "durationMs": 859434, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 20498, "promptTokensPerSec": null, "peakRssMb": 12816, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 126180, "output": 4819 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1161, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-craftbook-week-plan", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 1 }, "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T11-17-55-767Z-ruxb", "success": true, "includedInCapability": true, "durationMs": 353394, "timing": { "startedAt": "2026-10-28T11:17:55.768Z", "finishedAt": "2026-10-28T11:23:49.162Z", "timeToFirstArtifactMs": 47652, "timeToLastArtifactWriteMs": 331717, "timeToFirstTokenMs": 22274, "firstTurnTtftMs": 15511, "timeToFirstToolCallMs": 30823 }, "metrics": { "process": { "sampleCount": 69, "peakRssMb": 30235, "peakCpuPercent": 140.3, "firstAtMs": 0, "lastAtMs": 341680 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 69, "peakUsedMb": 99911, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 97121, "totalOutputTokens": 7479, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 98173, "todayTokensCached": 89953, "todayTokensOut": 1901, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 98173, "totalTokensCached": 89953, "totalTokensOut": 1901, "totalCost": 0, "medianOutputTokensPerSec": 91.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 91.4, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6110, "cachedInputTokens": 5268, "outputTokens": 104, "cost": 0, "durationMs": 14585, "outputTokensPerSec": 104.37471942960737, "at": "2026-10-28T11:23:27.488Z" }, "lastUpdated": "2026-10-28T11:23:27.488Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:23:39.056Z" } }, "source": "engine-log" }, "derived": { "durationMs": 342207, "meanTokensPerSec": 76.65, "genTokensPerSec": 76.65, "genTokens": 7479, "promptTokensPerSec": 1153.61, "peakRssMb": 30235, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 98173, "output": 1901 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-productivity-craftbook-ab-test-readout", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 1 }, "trialId": "craftbook-ab-test-readout-qwen3-8-27b-q4-2026-10-28T11-24-12-101Z-jr0t", "success": true, "includedInCapability": true, "durationMs": 1660499, "timing": { "startedAt": "2026-10-28T11:24:12.102Z", "finishedAt": "2026-10-28T11:51:52.601Z", "timeToFirstArtifactMs": 218925, "timeToLastArtifactWriteMs": 1655436, "timeToFirstTokenMs": 34619, "firstTurnTtftMs": 27963, "timeToFirstToolCallMs": 78039 }, "metrics": { "process": { "sampleCount": 331, "peakRssMb": 48322, "peakCpuPercent": 148.3, "firstAtMs": 0, "lastAtMs": 1656678 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 331, "peakUsedMb": 111612, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 97935, "totalOutputTokens": 35591, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 70694, "todayTokensCached": 66602, "todayTokensOut": 2924, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 70694, "totalTokensCached": 66602, "totalTokensOut": 2924, "totalCost": 0, "medianOutputTokensPerSec": 29.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.9, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8423, "cachedInputTokens": 8081, "outputTokens": 70, "cost": 0, "durationMs": 27824, "outputTokensPerSec": 47.686743707250386, "at": "2026-10-28T11:51:17.038Z" }, "lastUpdated": "2026-10-28T11:51:17.038Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:51:50.361Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1657205, "meanTokensPerSec": 27.24, "genTokensPerSec": 27.24, "genTokens": 35591, "promptTokensPerSec": 412.36, "peakRssMb": 48322, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 70694, "output": 2924 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-craftbook-ab-test-readout", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 1 }, "trialId": "craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-28T11-52-16-787Z-zk09", "success": true, "includedInCapability": true, "durationMs": 1269018, "timing": { "startedAt": "2026-10-28T11:52:16.787Z", "finishedAt": "2026-10-28T12:13:25.805Z", "timeToFirstArtifactMs": 40868, "timeToLastArtifactWriteMs": 1253600, "timeToFirstTokenMs": 19584, "firstTurnTtftMs": 12831, "timeToFirstToolCallMs": 35081 }, "metrics": { "process": { "sampleCount": 251, "peakRssMb": 14596, "peakCpuPercent": 119.7, "firstAtMs": 0, "lastAtMs": 1256020 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 251, "peakUsedMb": 80943, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 161018, "totalOutputTokens": 32047, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 198248, "todayTokensCached": 191670, "todayTokensOut": 6126, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 198248, "totalTokensCached": 191670, "totalTokensOut": 6126, "totalCost": 0, "medianOutputTokensPerSec": 35.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.5, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10612, "cachedInputTokens": 10413, "outputTokens": 108, "cost": 0, "durationMs": 25067, "outputTokensPerSec": 48.74530756225451, "at": "2026-10-28T12:12:49.048Z" }, "lastUpdated": "2026-10-28T12:12:49.048Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:13:14.603Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1256549, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 32047, "promptTokensPerSec": null, "peakRssMb": 14596, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 198248, "output": 6126 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1141, "sampleCount": 19, "structuredSampleCount": 0, "firstTokenSampleCount": 19, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-craftbook-ab-test-readout", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 1 }, "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T12-13-50-114Z-vius", "success": false, "includedInCapability": true, "durationMs": 4987570, "timing": { "startedAt": "2026-10-28T12:13:50.115Z", "finishedAt": "2026-10-28T13:36:57.685Z", "timeToFirstArtifactMs": 26069, "timeToLastArtifactWriteMs": 4677726, "timeToFirstTokenMs": 18135, "firstTurnTtftMs": 11368, "timeToFirstToolCallMs": 23016 }, "metrics": { "process": { "sampleCount": 992, "peakRssMb": 33778, "peakCpuPercent": 155.3, "firstAtMs": 0, "lastAtMs": 4981415 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 992, "peakUsedMb": 101450, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 773676, "totalOutputTokens": 135331, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 80, "todayTokensIn": 923809, "todayTokensCached": 833158, "todayTokensOut": 37335, "todayCost": 0, "totalTurns": 80, "totalTokensIn": 923809, "totalTokensCached": 833158, "totalTokensOut": 37335, "totalCost": 0, "medianOutputTokensPerSec": 83.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 83.4, "turns": 80 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11756, "cachedInputTokens": 11194, "outputTokens": 361, "cost": 0, "durationMs": 59736, "outputTokensPerSec": 75.38297693239028, "at": "2026-10-28T13:31:47.841Z" }, "lastUpdated": "2026-10-28T13:31:47.841Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T13:36:57.409Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4986253, "meanTokensPerSec": 74.54, "genTokensPerSec": 74.54, "genTokens": 135331, "promptTokensPerSec": 1072.34, "peakRssMb": 33778, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 923809, "output": 37335 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-craftbook-ab-test-readout", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 1 }, "trialId": "craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-28T13-37-19-431Z-r3n3", "success": true, "includedInCapability": true, "durationMs": 1329423, "timing": { "startedAt": "2026-10-28T13:37:19.431Z", "finishedAt": "2026-10-28T13:59:28.854Z", "timeToFirstArtifactMs": 78204, "timeToLastArtifactWriteMs": 1319163, "timeToFirstTokenMs": 28536, "firstTurnTtftMs": 22067, "timeToFirstToolCallMs": 42186 }, "metrics": { "process": { "sampleCount": 264, "peakRssMb": 19646, "peakCpuPercent": 113, "firstAtMs": 0, "lastAtMs": 1321323 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 264, "peakUsedMb": 96270, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 98948, "totalOutputTokens": 26069, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 91531, "todayTokensCached": 87326, "todayTokensOut": 5610, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 91531, "totalTokensCached": 87326, "totalTokensOut": 5610, "totalCost": 0, "medianOutputTokensPerSec": 25.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.9, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8677, "cachedInputTokens": 8588, "outputTokens": 68, "cost": 0, "durationMs": 39783, "outputTokensPerSec": 25.92245908231031, "at": "2026-10-28T13:56:41.049Z" }, "lastUpdated": "2026-10-28T13:56:41.049Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T13:59:23.495Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1323106, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 26069, "promptTokensPerSec": null, "peakRssMb": 19646, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 91531, "output": 5610 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 504, "sampleCount": 8, "structuredSampleCount": 0, "firstTokenSampleCount": 8, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-productivity-craftbook-annotated-bibliography", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 1 }, "trialId": "craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-28T13-59-53-091Z-9lcu", "success": true, "includedInCapability": true, "durationMs": 783068, "timing": { "startedAt": "2026-10-28T13:59:53.092Z", "finishedAt": "2026-10-28T14:12:56.160Z", "timeToFirstArtifactMs": 58026, "timeToLastArtifactWriteMs": 773465, "timeToFirstTokenMs": 21868, "firstTurnTtftMs": 15316, "timeToFirstToolCallMs": 48171 }, "metrics": { "process": { "sampleCount": 155, "peakRssMb": 14097, "peakCpuPercent": 145, "firstAtMs": 0, "lastAtMs": 774171 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 155, "peakUsedMb": 89525, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 140868, "totalOutputTokens": 15669, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 133732, "todayTokensCached": 129465, "todayTokensOut": 3527, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 133732, "totalTokensCached": 129465, "totalTokensOut": 3527, "totalCost": 0, "medianOutputTokensPerSec": 35.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.5, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11374, "cachedInputTokens": 10693, "outputTokens": 159, "cost": 0, "durationMs": 26766, "outputTokensPerSec": 35.85903325561603, "at": "2026-10-28T14:12:46.557Z" }, "lastUpdated": "2026-10-28T14:12:46.557Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:12:48.864Z" } }, "source": "engine-log" }, "derived": { "durationMs": 774724, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 15669, "promptTokensPerSec": null, "peakRssMb": 14097, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 133732, "output": 3527 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1099, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-productivity-craftbook-annotated-bibliography", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 1 }, "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T14-13-18-319Z-l8fk", "success": true, "includedInCapability": true, "durationMs": 154280, "timing": { "startedAt": "2026-10-28T14:13:18.319Z", "finishedAt": "2026-10-28T14:15:52.599Z", "timeToFirstArtifactMs": 29199, "timeToLastArtifactWriteMs": 137792, "timeToFirstTokenMs": 18157, "firstTurnTtftMs": 11440, "timeToFirstToolCallMs": 23464 }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 25198, "peakCpuPercent": 126.4, "firstAtMs": 0, "lastAtMs": 140594 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 100496, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 43228, "totalOutputTokens": 4553, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43200, "todayTokensCached": 41407, "todayTokensOut": 1395, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43200, "totalTokensCached": 41407, "totalTokensOut": 1395, "totalCost": 0, "medianOutputTokensPerSec": 80.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 80.9, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10964, "cachedInputTokens": 10711, "outputTokens": 135, "cost": 0, "durationMs": 26273, "outputTokensPerSec": 84.30732410440771, "at": "2026-10-28T14:15:36.111Z" }, "lastUpdated": "2026-10-28T14:15:36.111Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:15:40.493Z" } }, "source": "engine-log" }, "derived": { "durationMs": 141123, "meanTokensPerSec": 76.54, "genTokensPerSec": 76.54, "genTokens": 4553, "promptTokensPerSec": 1040.91, "peakRssMb": 25198, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43200, "output": 1395 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-craftbook-annotated-bibliography", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 1 }, "trialId": "craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-28T14-16-16-509Z-tygs", "success": true, "includedInCapability": true, "durationMs": 352807, "timing": { "startedAt": "2026-10-28T14:16:16.510Z", "finishedAt": "2026-10-28T14:22:09.317Z", "timeToFirstArtifactMs": 48417, "timeToLastArtifactWriteMs": 339753, "timeToFirstTokenMs": 23773, "firstTurnTtftMs": 17251, "timeToFirstToolCallMs": 24714 }, "metrics": { "process": { "sampleCount": 69, "peakRssMb": 19116, "peakCpuPercent": 127.3, "firstAtMs": 0, "lastAtMs": 341462 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 69, "peakUsedMb": 103911, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 45154, "totalOutputTokens": 6513, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 45026, "todayTokensCached": 42798, "todayTokensOut": 5057, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 45026, "totalTokensCached": 42798, "totalTokensOut": 5057, "totalCost": 0, "medianOutputTokensPerSec": 28.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.3, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12527, "cachedInputTokens": 12096, "outputTokens": 1182, "cost": 0, "durationMs": 83119, "outputTokensPerSec": 23.998892393539677, "at": "2026-10-28T14:21:56.263Z" }, "lastUpdated": "2026-10-28T14:21:56.263Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:21:59.567Z" } }, "source": "engine-log" }, "derived": { "durationMs": 341990, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 6513, "promptTokensPerSec": null, "peakRssMb": 19116, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 45026, "output": 5057 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 647, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-craftbook-annotated-bibliography", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 1 }, "trialId": "craftbook-annotated-bibliography-qwen3-8-27b-q4-2026-10-28T14-22-31-452Z-z2rx", "success": true, "includedInCapability": true, "durationMs": 532958, "timing": { "startedAt": "2026-10-28T14:22:31.452Z", "finishedAt": "2026-10-28T14:31:24.410Z", "timeToFirstArtifactMs": 69961, "timeToLastArtifactWriteMs": 527300, "timeToFirstTokenMs": 31932, "firstTurnTtftMs": 25228, "timeToFirstToolCallMs": 40488 }, "metrics": { "process": { "sampleCount": 106, "peakRssMb": 45043, "peakCpuPercent": 94.3, "firstAtMs": 0, "lastAtMs": 527229 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 106, "peakUsedMb": 119014, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47990, "totalOutputTokens": 10456, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 48162, "todayTokensCached": 47108, "todayTokensOut": 3770, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 48162, "totalTokensCached": 47108, "totalTokensOut": 3770, "totalCost": 0, "medianOutputTokensPerSec": 29.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.7, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13134, "cachedInputTokens": 12713, "outputTokens": 834, "cost": 0, "durationMs": 80597, "outputTokensPerSec": 24.2340597053918, "at": "2026-10-28T14:31:18.752Z" }, "lastUpdated": "2026-10-28T14:31:18.752Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:31:20.278Z" } }, "source": "engine-log" }, "derived": { "durationMs": 527777, "meanTokensPerSec": 28.27, "genTokensPerSec": 28.27, "genTokens": 10456, "promptTokensPerSec": 406.81, "peakRssMb": 45043, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 48162, "output": 3770 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-productivity-craftbook-research-to-document", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 1 }, "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T14-31-46-983Z-5x3d", "success": true, "includedInCapability": true, "durationMs": 239654, "timing": { "startedAt": "2026-10-28T14:31:46.983Z", "finishedAt": "2026-10-28T14:35:46.637Z", "timeToFirstArtifactMs": 28763, "timeToLastArtifactWriteMs": 176132, "timeToFirstTokenMs": 18773, "firstTurnTtftMs": 12092, "timeToFirstToolCallMs": 21641 }, "metrics": { "process": { "sampleCount": 46, "peakRssMb": 30375, "peakCpuPercent": 126.4, "firstAtMs": 0, "lastAtMs": 225944 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 103357, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 95662, "totalOutputTokens": 5082, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 72418, "todayTokensCached": 71360, "todayTokensOut": 714, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 72418, "totalTokensCached": 71360, "totalTokensOut": 714, "totalCost": 0, "medianOutputTokensPerSec": 69.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 69.7, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19336, "cachedInputTokens": 18984, "outputTokens": 35, "cost": 0, "durationMs": 40033, "outputTokensPerSec": 69.70412648428787, "at": "2026-10-28T14:35:29.076Z" }, "lastUpdated": "2026-10-28T14:35:29.076Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:35:34.494Z" } }, "source": "engine-log" }, "derived": { "durationMs": 226467, "meanTokensPerSec": 68.2, "genTokensPerSec": 68.2, "genTokens": 5082, "promptTokensPerSec": 938.9, "peakRssMb": 30375, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 72418, "output": 714 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-productivity-craftbook-research-to-document", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 1 }, "trialId": "craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-28T14-36-12-945Z-5kbv", "success": true, "includedInCapability": true, "durationMs": 952845, "timing": { "startedAt": "2026-10-28T14:36:12.945Z", "finishedAt": "2026-10-28T14:52:05.790Z", "timeToFirstArtifactMs": 49047, "timeToLastArtifactWriteMs": 299556, "timeToFirstTokenMs": 27001, "firstTurnTtftMs": 20399, "timeToFirstToolCallMs": 28012 }, "metrics": { "process": { "sampleCount": 188, "peakRssMb": 19970, "peakCpuPercent": 150.6, "firstAtMs": 0, "lastAtMs": 939126 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 188, "peakUsedMb": 109084, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 123040, "totalOutputTokens": 6631, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 94401, "todayTokensCached": 91438, "todayTokensOut": 1760, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 94401, "totalTokensCached": 91438, "totalTokensOut": 1760, "totalCost": 0, "medianOutputTokensPerSec": 24.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.9, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 31785, "cachedInputTokens": 29367, "outputTokens": 78, "cost": 0, "durationMs": 252735, "outputTokensPerSec": 23.55148010873952, "at": "2026-10-28T14:51:48.174Z" }, "lastUpdated": "2026-10-28T14:51:48.174Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:51:53.655Z" } }, "source": "engine-log" }, "derived": { "durationMs": 939653, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 6631, "promptTokensPerSec": null, "peakRssMb": 19970, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 94401, "output": 1760 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 494, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-productivity-craftbook-research-to-document", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 1 }, "trialId": "craftbook-research-to-document-qwen3-8-27b-q4-2026-10-28T14-52-29-919Z-r2ty", "success": true, "includedInCapability": true, "durationMs": 686548, "timing": { "startedAt": "2026-10-28T14:52:29.919Z", "finishedAt": "2026-10-28T15:03:56.467Z", "timeToFirstArtifactMs": 63566, "timeToLastArtifactWriteMs": 250326, "timeToFirstTokenMs": 31258, "firstTurnTtftMs": 24566, "timeToFirstToolCallMs": 32802 }, "metrics": { "process": { "sampleCount": 135, "peakRssMb": 47297, "peakCpuPercent": 117.9, "firstAtMs": 0, "lastAtMs": 672858 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 135, "peakUsedMb": 119542, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 130911, "totalOutputTokens": 7073, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 101796, "todayTokensCached": 100057, "todayTokensOut": 2099, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 101796, "totalTokensCached": 100057, "totalTokensOut": 2099, "totalCost": 0, "medianOutputTokensPerSec": 29.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.7, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 36047, "cachedInputTokens": 35361, "outputTokens": 78, "cost": 0, "durationMs": 182580, "outputTokensPerSec": 29.05186777098979, "at": "2026-10-28T15:03:39.810Z" }, "lastUpdated": "2026-10-28T15:03:39.810Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:03:44.350Z" } }, "source": "engine-log" }, "derived": { "durationMs": 673385, "meanTokensPerSec": 27.39, "genTokensPerSec": 27.39, "genTokens": 7073, "promptTokensPerSec": 353.06, "peakRssMb": 47297, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 101796, "output": 2099 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-productivity-craftbook-research-to-document", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 1 }, "trialId": "craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-28T15-04-18-199Z-9l8o", "success": true, "includedInCapability": true, "durationMs": 2228144, "timing": { "startedAt": "2026-10-28T15:04:18.200Z", "finishedAt": "2026-10-28T15:41:26.344Z", "timeToFirstArtifactMs": 31553, "timeToLastArtifactWriteMs": 1232846, "timeToFirstTokenMs": 19916, "firstTurnTtftMs": 13403, "timeToFirstToolCallMs": 24998 }, "metrics": { "process": { "sampleCount": 443, "peakRssMb": 12496, "peakCpuPercent": 115.8, "firstAtMs": 0, "lastAtMs": 2220958 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 443, "peakUsedMb": 131010, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 251676, "totalOutputTokens": 36560, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 243104, "todayTokensCached": 240671, "todayTokensOut": 2112, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 243104, "totalTokensCached": 240671, "totalTokensOut": 2112, "totalCost": 0, "medianOutputTokensPerSec": 35.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.3, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17891, "cachedInputTokens": 17755, "outputTokens": 35, "cost": 0, "durationMs": 48472, "outputTokensPerSec": 34.25073394049706, "at": "2026-10-28T15:41:19.511Z" }, "lastUpdated": "2026-10-28T15:41:19.511Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:41:21.009Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2221739, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 36560, "promptTokensPerSec": null, "peakRssMb": 12496, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 243104, "output": 2112 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1084, "sampleCount": 16, "structuredSampleCount": 0, "firstTokenSampleCount": 16, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-developer-fix-squisq-bugs", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 1 }, "trialId": "fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-28T15-41-48-693Z-tr26", "success": false, "includedInCapability": true, "durationMs": 510679, "timing": { "startedAt": "2026-10-28T15:41:48.693Z", "finishedAt": "2026-10-28T15:50:19.372Z", "timeToFirstArtifactMs": 45251, "timeToLastArtifactWriteMs": 504567, "timeToFirstTokenMs": 15431, "firstTurnTtftMs": 13755, "timeToFirstToolCallMs": 17987 }, "metrics": { "process": { "sampleCount": 102, "peakRssMb": 18021, "peakCpuPercent": 123.1, "firstAtMs": 0, "lastAtMs": 507167 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 102, "peakUsedMb": 87300, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 82568, "totalOutputTokens": 8674, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 74597, "todayTokensCached": 27081, "todayTokensOut": 5269, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 74597, "totalTokensCached": 27081, "totalTokensOut": 5269, "totalCost": 0, "medianOutputTokensPerSec": 30.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.5, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 17607, "cachedInputTokens": 0, "outputTokens": 1718, "cost": 0, "durationMs": 91016, "outputTokensPerSec": 30.451089303901938, "at": "2026-10-28T15:50:13.260Z" }, "lastUpdated": "2026-10-28T15:50:13.260Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:50:17.476Z" } }, "source": "engine-log" }, "derived": { "durationMs": 507716, "meanTokensPerSec": 30, "genTokensPerSec": 30, "genTokens": 8674, "promptTokensPerSec": null, "peakRssMb": 18021, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 74597, "output": 5269 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 520, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-fix-squisq-bugs", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 1 }, "trialId": "fix-squisq-bugs-qwen3-8-27b-q4-2026-10-28T15-50-43-095Z-lwml", "success": true, "includedInCapability": true, "durationMs": 90385, "timing": { "startedAt": "2026-10-28T15:50:43.096Z", "finishedAt": "2026-10-28T15:52:13.481Z", "timeToFirstArtifactMs": 49725, "timeToLastArtifactWriteMs": 85542, "timeToFirstTokenMs": 17114, "firstTurnTtftMs": 15293, "timeToFirstToolCallMs": 20890 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 37471, "peakCpuPercent": 96.5, "firstAtMs": 0, "lastAtMs": 85353 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 90900, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8571, "totalOutputTokens": 2334, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:52:10.276Z" } }, "source": "engine-log" }, "derived": { "durationMs": 86130, "meanTokensPerSec": 41.06, "genTokensPerSec": 41.06, "genTokens": 2334, "promptTokensPerSec": 425.24, "peakRssMb": 37471, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-fix-squisq-bugs", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 1 }, "trialId": "fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-28T15-52-35-325Z-vkfu", "success": true, "includedInCapability": true, "durationMs": 143721, "timing": { "startedAt": "2026-10-28T15:52:35.325Z", "finishedAt": "2026-10-28T15:54:59.046Z", "timeToFirstArtifactMs": 48512, "timeToLastArtifactWriteMs": 137079, "timeToFirstTokenMs": 12722, "firstTurnTtftMs": 11033, "timeToFirstToolCallMs": 19555 }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 10212, "peakCpuPercent": 117.7, "firstAtMs": 0, "lastAtMs": 140650 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 70959, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16672, "totalOutputTokens": 3868, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:54:57.819Z" } }, "source": "engine-log" }, "derived": { "durationMs": 141432, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 3868, "promptTokensPerSec": null, "peakRssMb": 10212, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1013, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-fix-squisq-bugs", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 1 }, "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T15-55-22-422Z-km1u", "success": true, "includedInCapability": true, "durationMs": 82209, "timing": { "startedAt": "2026-10-28T15:55:22.423Z", "finishedAt": "2026-10-28T15:56:44.632Z", "timeToFirstArtifactMs": 24368, "timeToLastArtifactWriteMs": 68646, "timeToFirstTokenMs": 10003, "firstTurnTtftMs": 8155, "timeToFirstToolCallMs": 13934 }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 20050, "peakCpuPercent": 99.6, "firstAtMs": 0, "lastAtMs": 70291 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 71850, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 14435, "totalOutputTokens": 4254, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:56:34.538Z" } }, "source": "engine-log" }, "derived": { "durationMs": 71067, "meanTokensPerSec": 91.71, "genTokensPerSec": 91.71, "genTokens": 4254, "promptTokensPerSec": 880.83, "peakRssMb": 20050, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-developer-dev-craftbook-routing", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 1 }, "trialId": "dev-craftbook-routing-qwen3-8-27b-q4-2026-10-28T15-57-05-287Z-82ze", "success": true, "includedInCapability": true, "durationMs": 90499, "timing": { "startedAt": "2026-10-28T15:57:05.288Z", "finishedAt": "2026-10-28T15:58:35.787Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 34977, "firstTurnTtftMs": 33068, "timeToFirstToolCallMs": 62015 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 37408, "peakCpuPercent": 118.9, "firstAtMs": 1, "lastAtMs": 85356 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 91967, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13092, "totalOutputTokens": 1454, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:58:32.471Z" } }, "source": "engine-log" }, "derived": { "durationMs": 86133, "meanTokensPerSec": 30.78, "genTokensPerSec": 30.78, "genTokens": 1454, "promptTokensPerSec": 427.64, "peakRssMb": 37408, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-dev-craftbook-routing", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 1 }, "trialId": "dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-28T15-58-57-938Z-0lmg", "success": false, "includedInCapability": true, "durationMs": 508034, "timing": { "startedAt": "2026-10-28T15:58:57.938Z", "finishedAt": "2026-10-28T16:07:25.972Z", "timeToFirstArtifactMs": 436076, "timeToLastArtifactWriteMs": 504165, "timeToFirstTokenMs": 17422, "firstTurnTtftMs": 15715, "timeToFirstToolCallMs": 33303 }, "metrics": { "process": { "sampleCount": 101, "peakRssMb": 11240, "peakCpuPercent": 102.6, "firstAtMs": 0, "lastAtMs": 502669 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 101, "peakUsedMb": 73248, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 55688, "totalOutputTokens": 14035, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 82028, "todayTokensCached": 59866, "todayTokensOut": 12446, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 82028, "totalTokensCached": 59866, "totalTokensOut": 12446, "totalCost": 0, "medianOutputTokensPerSec": 35, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9019, "cachedInputTokens": 1629, "outputTokens": 64, "cost": 0, "durationMs": 18736, "outputTokensPerSec": 43.292193829598105, "at": "2026-10-28T16:07:22.106Z" }, "lastUpdated": "2026-10-28T16:07:22.106Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:07:23.959Z" } }, "source": "engine-log" }, "derived": { "durationMs": 504964, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 14035, "promptTokensPerSec": null, "peakRssMb": 11240, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 82028, "output": 12446 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 741.5, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-dev-craftbook-routing", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 1 }, "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T16-07-47-451Z-g099", "success": true, "includedInCapability": true, "durationMs": 124669, "timing": { "startedAt": "2026-10-28T16:07:47.452Z", "finishedAt": "2026-10-28T16:09:52.121Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 16069, "firstTurnTtftMs": 14175, "timeToFirstToolCallMs": 40781 }, "metrics": { "process": { "sampleCount": 23, "peakRssMb": 23597, "peakCpuPercent": 117.1, "firstAtMs": 0, "lastAtMs": 110470 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 75863, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13214, "totalOutputTokens": 5287, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 47299, "todayTokensCached": 46527, "todayTokensOut": 1336, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 47299, "totalTokensCached": 46527, "totalTokensOut": 1336, "totalCost": 0, "medianOutputTokensPerSec": 58.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 58.3, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13164, "cachedInputTokens": 12966, "outputTokens": 34, "cost": 0, "durationMs": 19922, "outputTokensPerSec": 61.40540218677662, "at": "2026-10-28T16:09:39.043Z" }, "lastUpdated": "2026-10-28T16:09:39.043Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:09:40.002Z" } }, "source": "engine-log" }, "derived": { "durationMs": 111500, "meanTokensPerSec": 60.03, "genTokensPerSec": 60.03, "genTokens": 5287, "promptTokensPerSec": 794.05, "peakRssMb": 23597, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 47299, "output": 1336 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-dev-craftbook-routing", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 1 }, "trialId": "dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-28T16-10-17-744Z-jvwm", "success": false, "includedInCapability": true, "durationMs": 861397, "timing": { "startedAt": "2026-10-28T16:10:17.745Z", "finishedAt": "2026-10-28T16:24:39.142Z", "timeToFirstArtifactMs": 595399, "timeToLastArtifactWriteMs": 595399, "timeToFirstTokenMs": 28834, "firstTurnTtftMs": 27128, "timeToFirstToolCallMs": 49090 }, "metrics": { "process": { "sampleCount": 171, "peakRssMb": 18777, "peakCpuPercent": 129.5, "firstAtMs": 0, "lastAtMs": 854771 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 171, "peakUsedMb": 88519, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 27909, "totalOutputTokens": 13197, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 21886, "todayTokensCached": 17225, "todayTokensOut": 2853, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 21886, "totalTokensCached": 17225, "totalTokensOut": 2853, "totalCost": 0, "medianOutputTokensPerSec": 27.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27.6, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4372, "cachedInputTokens": 3518, "outputTokens": 1738, "cost": 0, "durationMs": 84980, "outputTokensPerSec": 27.572183596669124, "at": "2026-10-28T16:22:39.272Z" }, "lastUpdated": "2026-10-28T16:22:39.272Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:24:36.641Z" } }, "source": "engine-log" }, "derived": { "durationMs": 857829, "meanTokensPerSec": 16, "genTokensPerSec": 16, "genTokens": 13197, "promptTokensPerSec": null, "peakRssMb": 18777, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21886, "output": 2853 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 524.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-developer-interface-contract", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "interface-contract", "repetition": 1 }, "trialId": "interface-contract-mlx-gemma4-12b-q4-2026-10-28T16-25-00-876Z-0q67", "success": true, "includedInCapability": true, "durationMs": 68421, "timing": { "startedAt": "2026-10-28T16:25:00.876Z", "finishedAt": "2026-10-28T16:26:09.297Z", "timeToFirstArtifactMs": 55925, "timeToLastArtifactWriteMs": 65543, "timeToFirstTokenMs": 17501, "firstTurnTtftMs": 15807, "timeToFirstToolCallMs": 38235 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 10825, "peakCpuPercent": 116.6, "firstAtMs": 0, "lastAtMs": 65298 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 73887, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18469, "totalOutputTokens": 977, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 10676, "todayTokensCached": 10495, "todayTokensOut": 24, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 10676, "totalTokensCached": 10495, "totalTokensOut": 24, "totalCost": 0, "medianOutputTokensPerSec": 42.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10676, "cachedInputTokens": 10495, "outputTokens": 24, "cost": 0, "durationMs": 38395, "outputTokensPerSec": 42.2001894489159, "at": "2026-10-28T16:25:40.965Z" }, "lastUpdated": "2026-10-28T16:25:40.965Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:26:07.770Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65829, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 977, "promptTokensPerSec": null, "peakRssMb": 10825, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 10676, "output": 24 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1148, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-interface-contract", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "interface-contract", "repetition": 1 }, "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T16-26-31-568Z-4cjw", "success": true, "includedInCapability": true, "durationMs": 56862, "timing": { "startedAt": "2026-10-28T16:26:31.568Z", "finishedAt": "2026-10-28T16:27:28.430Z", "timeToFirstArtifactMs": 40494, "timeToLastArtifactWriteMs": 44422, "timeToFirstTokenMs": 11573, "firstTurnTtftMs": 9663, "timeToFirstToolCallMs": 20938 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 20242, "peakCpuPercent": 116.1, "firstAtMs": 1, "lastAtMs": 45186 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 77799, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15332, "totalOutputTokens": 1722, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7324, "todayTokensCached": 7156, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7324, "totalTokensCached": 7156, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 63.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 63.1, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7324, "cachedInputTokens": 7156, "outputTokens": 23, "cost": 0, "durationMs": 20679, "outputTokensPerSec": 63.072671184963475, "at": "2026-10-28T16:26:54.157Z" }, "lastUpdated": "2026-10-28T16:26:54.157Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:27:18.332Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45711, "meanTokensPerSec": 87.8, "genTokensPerSec": 87.8, "genTokens": 1722, "promptTokensPerSec": 1185.32, "peakRssMb": 20242, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7324, "output": 23 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-interface-contract", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "interface-contract", "repetition": 1 }, "trialId": "interface-contract-mlx-qwen3-8-27b-q4-2026-10-28T16-27-50-316Z-2l0n", "success": true, "includedInCapability": true, "durationMs": 174277, "timing": { "startedAt": "2026-10-28T16:27:50.316Z", "finishedAt": "2026-10-28T16:30:44.593Z", "timeToFirstArtifactMs": 167513, "timeToLastArtifactWriteMs": 167531, "timeToFirstTokenMs": 26041, "firstTurnTtftMs": 24348, "timeToFirstToolCallMs": 34528 }, "metrics": { "process": { "sampleCount": 35, "peakRssMb": 18243, "peakCpuPercent": 121.2, "firstAtMs": 0, "lastAtMs": 170716 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 35, "peakUsedMb": 92066, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 19608, "totalOutputTokens": 3385, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 24742, "todayTokensCached": 24347, "todayTokensOut": 260, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 24742, "totalTokensCached": 24347, "totalTokensOut": 260, "totalCost": 0, "medianOutputTokensPerSec": 36.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.5, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12431, "cachedInputTokens": 12295, "outputTokens": 108, "cost": 0, "durationMs": 3269, "outputTokensPerSec": 38.390717909234894, "at": "2026-10-28T16:28:40.795Z" }, "lastUpdated": "2026-10-28T16:28:40.795Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:30:42.629Z" } }, "source": "engine-log" }, "derived": { "durationMs": 171243, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 3385, "promptTokensPerSec": null, "peakRssMb": 18243, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 24742, "output": 260 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 483.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-interface-contract", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "interface-contract", "repetition": 1 }, "trialId": "interface-contract-qwen3-8-27b-q4-2026-10-28T16-31-08-286Z-0vgk", "success": true, "includedInCapability": true, "durationMs": 130336, "timing": { "startedAt": "2026-10-28T16:31:08.287Z", "finishedAt": "2026-10-28T16:33:18.623Z", "timeToFirstArtifactMs": 125533, "timeToLastArtifactWriteMs": 125549, "timeToFirstTokenMs": 24717, "firstTurnTtftMs": 22865, "timeToFirstToolCallMs": 51042 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 39112, "peakCpuPercent": 112.8, "firstAtMs": 0, "lastAtMs": 125519 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 96435, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 14881, "totalOutputTokens": 2417, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7646, "todayTokensCached": 7098, "todayTokensOut": 111, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7646, "totalTokensCached": 7098, "totalTokensOut": 111, "totalCost": 0, "medianOutputTokensPerSec": 26.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.7, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7646, "cachedInputTokens": 7098, "outputTokens": 111, "cost": 0, "durationMs": 55141, "outputTokensPerSec": 26.671813518438952, "at": "2026-10-28T16:32:05.280Z" }, "lastUpdated": "2026-10-28T16:32:05.280Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:33:15.380Z" } }, "source": "engine-log" }, "derived": { "durationMs": 126045, "meanTokensPerSec": 30.86, "genTokensPerSec": 30.86, "genTokens": 2417, "promptTokensPerSec": 455.2, "peakRssMb": 39112, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7646, "output": 111 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-developer-craftbook-code-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 1 }, "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T16-33-41-029Z-o06u", "success": true, "includedInCapability": true, "durationMs": 143801, "timing": { "startedAt": "2026-10-28T16:33:41.030Z", "finishedAt": "2026-10-28T16:36:04.831Z", "timeToFirstArtifactMs": 84717, "timeToLastArtifactWriteMs": 84717, "timeToFirstTokenMs": 17655, "firstTurnTtftMs": 10925, "timeToFirstToolCallMs": 20088 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 21809, "peakCpuPercent": 114.4, "firstAtMs": 0, "lastAtMs": 135583 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 78797, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 67255, "totalOutputTokens": 3571, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 32911, "todayTokensCached": 21371, "todayTokensOut": 510, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 32911, "totalTokensCached": 21371, "totalTokensOut": 510, "totalCost": 0, "medianOutputTokensPerSec": 76.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 76.8, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11857, "cachedInputTokens": 11660, "outputTokens": 33, "cost": 0, "durationMs": 28263, "outputTokensPerSec": 94.81200557020533, "at": "2026-10-28T16:35:35.952Z" }, "lastUpdated": "2026-10-28T16:35:35.952Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:35:58.189Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136109, "meanTokensPerSec": 74.63, "genTokensPerSec": 74.63, "genTokens": 3571, "promptTokensPerSec": 1037.82, "peakRssMb": 21809, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32911, "output": 510 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-craftbook-code-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 1 }, "trialId": "craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-28T16-36-28-242Z-f2ak", "success": true, "includedInCapability": true, "durationMs": 219576, "timing": { "startedAt": "2026-10-28T16:36:28.243Z", "finishedAt": "2026-10-28T16:40:07.819Z", "timeToFirstArtifactMs": 208398, "timeToLastArtifactWriteMs": 208398, "timeToFirstTokenMs": 25569, "firstTurnTtftMs": 18983, "timeToFirstToolCallMs": 26926 }, "metrics": { "process": { "sampleCount": 43, "peakRssMb": 18001, "peakCpuPercent": 121.4, "firstAtMs": 0, "lastAtMs": 210884 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 43, "peakUsedMb": 88644, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 21482, "totalOutputTokens": 4001, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 21402, "todayTokensCached": 21056, "todayTokensOut": 2033, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 21402, "totalTokensCached": 21056, "totalTokensOut": 2033, "totalCost": 0, "medianOutputTokensPerSec": 29.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.7, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11376, "cachedInputTokens": 11116, "outputTokens": 1917, "cost": 0, "durationMs": 121769, "outputTokensPerSec": 27.64781997677749, "at": "2026-10-28T16:39:56.641Z" }, "lastUpdated": "2026-10-28T16:39:56.641Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:40:00.723Z" } }, "source": "engine-log" }, "derived": { "durationMs": 211410, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 4001, "promptTokensPerSec": null, "peakRssMb": 18001, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21402, "output": 2033 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 599, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-craftbook-code-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 1 }, "trialId": "craftbook-code-review-qwen3-8-27b-q4-2026-10-28T16-40-29-296Z-3a9o", "success": true, "includedInCapability": true, "durationMs": 337657, "timing": { "startedAt": "2026-10-28T16:40:29.297Z", "finishedAt": "2026-10-28T16:46:06.954Z", "timeToFirstArtifactMs": 240830, "timeToLastArtifactWriteMs": 240830, "timeToFirstTokenMs": 29760, "firstTurnTtftMs": 23061, "timeToFirstToolCallMs": 30680 }, "metrics": { "process": { "sampleCount": 67, "peakRssMb": 42352, "peakCpuPercent": 110.7, "firstAtMs": 1, "lastAtMs": 331391 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 67, "peakUsedMb": 100583, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47871, "totalOutputTokens": 5704, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 34032, "todayTokensCached": 21762, "todayTokensOut": 4147, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 34032, "totalTokensCached": 21762, "totalTokensOut": 4147, "totalCost": 0, "medianOutputTokensPerSec": 41.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 41.8, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12039, "cachedInputTokens": 11616, "outputTokens": 69, "cost": 0, "durationMs": 50019, "outputTokensPerSec": 41.84963058469472, "at": "2026-10-28T16:45:20.951Z" }, "lastUpdated": "2026-10-28T16:45:20.951Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:46:02.270Z" } }, "source": "engine-log" }, "derived": { "durationMs": 331921, "meanTokensPerSec": 29.75, "genTokensPerSec": 29.75, "genTokens": 5704, "promptTokensPerSec": 405.93, "peakRssMb": 42352, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 34032, "output": 4147 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-craftbook-code-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 1 }, "trialId": "craftbook-code-review-mlx-gemma4-12b-q4-2026-10-28T16-46-29-221Z-kxrz", "success": true, "includedInCapability": true, "durationMs": 129167, "timing": { "startedAt": "2026-10-28T16:46:29.222Z", "finishedAt": "2026-10-28T16:48:38.389Z", "timeToFirstArtifactMs": 88066, "timeToLastArtifactWriteMs": 88066, "timeToFirstTokenMs": 19840, "firstTurnTtftMs": 13294, "timeToFirstToolCallMs": 25849 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 10836, "peakCpuPercent": 117.5, "firstAtMs": 1, "lastAtMs": 125563 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 73382, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 38078, "totalOutputTokens": 1747, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 33076, "todayTokensCached": 32107, "todayTokensOut": 582, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 33076, "totalTokensCached": 32107, "totalTokensOut": 582, "totalCost": 0, "medianOutputTokensPerSec": 40.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.2, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11961, "cachedInputTokens": 11750, "outputTokens": 33, "cost": 0, "durationMs": 20175, "outputTokensPerSec": 40.22655929694378, "at": "2026-10-28T16:48:21.012Z" }, "lastUpdated": "2026-10-28T16:48:21.012Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:48:36.376Z" } }, "source": "engine-log" }, "derived": { "durationMs": 126092, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 1747, "promptTokensPerSec": null, "peakRssMb": 10836, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 33076, "output": 582 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1220, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-developer-craftbook-api-contract-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 1 }, "trialId": "craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-28T16-48-59-569Z-uudw", "success": true, "includedInCapability": true, "durationMs": 1240739, "timing": { "startedAt": "2026-10-28T16:48:59.570Z", "finishedAt": "2026-10-28T17:09:40.309Z", "timeToFirstArtifactMs": 304249, "timeToLastArtifactWriteMs": 1233959, "timeToFirstTokenMs": 27316, "firstTurnTtftMs": 20776, "timeToFirstToolCallMs": 46099 }, "metrics": { "process": { "sampleCount": 247, "peakRssMb": 18941, "peakCpuPercent": 113.6, "firstAtMs": 0, "lastAtMs": 1235171 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 247, "peakUsedMb": 90047, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 61055, "totalOutputTokens": 28134, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 69033, "todayTokensCached": 67898, "todayTokensOut": 11917, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 69033, "totalTokensCached": 67898, "totalTokensOut": 11917, "totalCost": 0, "medianOutputTokensPerSec": 25.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.5, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 21713, "cachedInputTokens": 21175, "outputTokens": 510, "cost": 0, "durationMs": 182104, "outputTokensPerSec": 24.27521811553688, "at": "2026-10-28T17:09:33.528Z" }, "lastUpdated": "2026-10-28T17:09:33.528Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:09:36.356Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1235719, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 28134, "promptTokensPerSec": null, "peakRssMb": 18941, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 69033, "output": 11917 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 538, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-craftbook-api-contract-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 1 }, "trialId": "craftbook-api-contract-review-qwen3-8-27b-q4-2026-10-28T17-10-02-484Z-w4i1", "success": true, "includedInCapability": true, "durationMs": 841559, "timing": { "startedAt": "2026-10-28T17:10:02.485Z", "finishedAt": "2026-10-28T17:24:04.044Z", "timeToFirstArtifactMs": 124814, "timeToLastArtifactWriteMs": 835278, "timeToFirstTokenMs": 31685, "firstTurnTtftMs": 24961, "timeToFirstToolCallMs": 41338 }, "metrics": { "process": { "sampleCount": 168, "peakRssMb": 44257, "peakCpuPercent": 149.6, "firstAtMs": 0, "lastAtMs": 838523 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 168, "peakUsedMb": 101372, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 72318, "totalOutputTokens": 18052, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 57596, "todayTokensCached": 45976, "todayTokensOut": 12343, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 57596, "totalTokensCached": 45976, "totalTokensOut": 12343, "totalCost": 0, "medianOutputTokensPerSec": 31.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.6, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 19732, "cachedInputTokens": 19283, "outputTokens": 569, "cost": 0, "durationMs": 173492, "outputTokensPerSec": 32.13922349485877, "at": "2026-10-28T17:23:57.763Z" }, "lastUpdated": "2026-10-28T17:23:57.763Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:24:02.611Z" } }, "source": "engine-log" }, "derived": { "durationMs": 839076, "meanTokensPerSec": 28.53, "genTokensPerSec": 28.53, "genTokens": 18052, "promptTokensPerSec": 423.63, "peakRssMb": 44257, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 57596, "output": 12343 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-craftbook-api-contract-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 1 }, "trialId": "craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-28T17-24-25-881Z-vzmh", "success": true, "includedInCapability": true, "durationMs": 688798, "timing": { "startedAt": "2026-10-28T17:24:25.882Z", "finishedAt": "2026-10-28T17:35:54.680Z", "timeToFirstArtifactMs": 322020, "timeToLastArtifactWriteMs": 651065, "timeToFirstTokenMs": 21318, "firstTurnTtftMs": 14786, "timeToFirstToolCallMs": 32734 }, "metrics": { "process": { "sampleCount": 137, "peakRssMb": 11835, "peakCpuPercent": 117, "firstAtMs": 0, "lastAtMs": 683435 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 137, "peakUsedMb": 73984, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47951, "totalOutputTokens": 20668, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 56265, "todayTokensCached": 54069, "todayTokensOut": 2420, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 56265, "totalTokensCached": 54069, "totalTokensOut": 2420, "totalCost": 0, "medianOutputTokensPerSec": 34.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.8, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7732, "cachedInputTokens": 7550, "outputTokens": 39, "cost": 0, "durationMs": 24154, "outputTokensPerSec": 45.977781015318826, "at": "2026-10-28T17:35:41.749Z" }, "lastUpdated": "2026-10-28T17:35:41.749Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:35:50.911Z" } }, "source": "engine-log" }, "derived": { "durationMs": 683962, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 20668, "promptTokensPerSec": null, "peakRssMb": 11835, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 56265, "output": 2420 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1085, "sampleCount": 8, "structuredSampleCount": 0, "firstTokenSampleCount": 8, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-craftbook-api-contract-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 1 }, "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T17-36-18-343Z-x14y", "success": true, "includedInCapability": true, "durationMs": 362216, "timing": { "startedAt": "2026-10-28T17:36:18.343Z", "finishedAt": "2026-10-28T17:42:20.559Z", "timeToFirstArtifactMs": 57353, "timeToLastArtifactWriteMs": 318126, "timeToFirstTokenMs": 19093, "firstTurnTtftMs": 12319, "timeToFirstToolCallMs": 25337 }, "metrics": { "process": { "sampleCount": 72, "peakRssMb": 23346, "peakCpuPercent": 164.9, "firstAtMs": 1, "lastAtMs": 356489 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 72, "peakUsedMb": 85713, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 97732, "totalOutputTokens": 12383, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 81319, "todayTokensCached": 65577, "todayTokensOut": 6078, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 81319, "totalTokensCached": 65577, "totalTokensOut": 6078, "totalCost": 0, "medianOutputTokensPerSec": 59.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 59.6, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7797, "cachedInputTokens": 7608, "outputTokens": 62, "cost": 0, "durationMs": 17561, "outputTokensPerSec": 56.770379635627236, "at": "2026-10-28T17:41:57.525Z" }, "lastUpdated": "2026-10-28T17:41:57.525Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:42:16.407Z" } }, "source": "engine-log" }, "derived": { "durationMs": 357015, "meanTokensPerSec": 63.24, "genTokensPerSec": 63.24, "genTokens": 12383, "promptTokensPerSec": 957.28, "peakRssMb": 23346, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 81319, "output": 6078 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-developer-craftbook-deep-security-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 1 }, "trialId": "craftbook-deep-security-review-qwen3-8-27b-q4-2026-10-28T17-42-42-393Z-hjjk", "success": true, "includedInCapability": true, "durationMs": 454758, "timing": { "startedAt": "2026-10-28T17:42:42.393Z", "finishedAt": "2026-10-28T17:50:17.151Z", "timeToFirstArtifactMs": 130938, "timeToLastArtifactWriteMs": 447247, "timeToFirstTokenMs": 28865, "firstTurnTtftMs": 27073, "timeToFirstToolCallMs": 32927 }, "metrics": { "process": { "sampleCount": 90, "peakRssMb": 42952, "peakCpuPercent": 113.5, "firstAtMs": 0, "lastAtMs": 446807 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 90, "peakUsedMb": 104368, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 54327, "totalOutputTokens": 7989, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43756, "todayTokensCached": 40960, "todayTokensOut": 1282, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43756, "totalTokensCached": 40960, "totalTokensOut": 1282, "totalCost": 0, "medianOutputTokensPerSec": 24.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.8, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9171, "cachedInputTokens": 8438, "outputTokens": 312, "cost": 0, "durationMs": 31390, "outputTokensPerSec": 37.271259628358806, "at": "2026-10-28T17:50:09.642Z" }, "lastUpdated": "2026-10-28T17:50:09.642Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:50:13.303Z" } }, "source": "engine-log" }, "derived": { "durationMs": 449870, "meanTokensPerSec": 27.75, "genTokensPerSec": 27.75, "genTokens": 7989, "promptTokensPerSec": 396.27, "peakRssMb": 42952, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43756, "output": 1282 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-craftbook-deep-security-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 1 }, "trialId": "craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-28T17-50-41-916Z-2i20", "success": false, "includedInCapability": true, "durationMs": 367543, "timing": { "startedAt": "2026-10-28T17:50:41.916Z", "finishedAt": "2026-10-28T17:56:49.459Z", "timeToFirstArtifactMs": 294164, "timeToLastArtifactWriteMs": 324813, "timeToFirstTokenMs": 15606, "firstTurnTtftMs": 13746, "timeToFirstToolCallMs": 22611 }, "metrics": { "process": { "sampleCount": 73, "peakRssMb": 10538, "peakCpuPercent": 110.3, "firstAtMs": 0, "lastAtMs": 361731 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 73, "peakUsedMb": 73972, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18285, "totalOutputTokens": 12141, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 20234, "todayTokensCached": 18858, "todayTokensOut": 8384, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 20234, "totalTokensCached": 18858, "totalTokensOut": 8384, "totalCost": 0, "medianOutputTokensPerSec": 38.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11169, "cachedInputTokens": 10819, "outputTokens": 192, "cost": 0, "durationMs": 71591, "outputTokensPerSec": 39.12471091516225, "at": "2026-10-28T17:55:50.754Z" }, "lastUpdated": "2026-10-28T17:55:50.754Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:56:47.972Z" } }, "source": "engine-log" }, "derived": { "durationMs": 364781, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 12141, "promptTokensPerSec": null, "peakRssMb": 10538, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 20234, "output": 8384 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 321, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-craftbook-deep-security-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 1 }, "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T17-57-11-139Z-x7ql", "success": false, "includedInCapability": true, "durationMs": 317223, "timing": { "startedAt": "2026-10-28T17:57:11.139Z", "finishedAt": "2026-10-28T18:02:28.362Z", "timeToFirstArtifactMs": 39039, "timeToLastArtifactWriteMs": 307656, "timeToFirstTokenMs": 13050, "firstTurnTtftMs": 11175, "timeToFirstToolCallMs": 14660 }, "metrics": { "process": { "sampleCount": 62, "peakRssMb": 24265, "peakCpuPercent": 107.5, "firstAtMs": 0, "lastAtMs": 306287 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 62, "peakUsedMb": 78308, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 95696, "totalOutputTokens": 13216, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 201461, "todayTokensCached": 161957, "todayTokensOut": 6586, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 201461, "totalTokensCached": 161957, "totalTokensOut": 6586, "totalCost": 0, "medianOutputTokensPerSec": 71.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 71.4, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 24071, "cachedInputTokens": 23517, "outputTokens": 56, "cost": 0, "durationMs": 23399, "outputTokensPerSec": 50.04308254469984, "at": "2026-10-28T18:02:18.795Z" }, "lastUpdated": "2026-10-28T18:02:18.795Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:02:21.015Z" } }, "source": "engine-log" }, "derived": { "durationMs": 308824, "meanTokensPerSec": 71.76, "genTokensPerSec": 71.76, "genTokens": 13216, "promptTokensPerSec": 933.27, "peakRssMb": 24265, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 201461, "output": 6586 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-craftbook-deep-security-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 1 }, "trialId": "craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-28T18-02-49-769Z-hfjl", "success": true, "includedInCapability": true, "durationMs": 458771, "timing": { "startedAt": "2026-10-28T18:02:49.770Z", "finishedAt": "2026-10-28T18:10:28.541Z", "timeToFirstArtifactMs": 203916, "timeToLastArtifactWriteMs": 452406, "timeToFirstTokenMs": 23746, "firstTurnTtftMs": 22067, "timeToFirstToolCallMs": 26752 }, "metrics": { "process": { "sampleCount": 91, "peakRssMb": 18256, "peakCpuPercent": 123.2, "firstAtMs": 1, "lastAtMs": 451899 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 91, "peakUsedMb": 90555, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 60534, "totalOutputTokens": 8285, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 46447, "todayTokensCached": 19471, "todayTokensOut": 2255, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 46447, "totalTokensCached": 19471, "totalTokensOut": 2255, "totalCost": 0, "medianOutputTokensPerSec": 35.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 35.1, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9380, "cachedInputTokens": 1982, "outputTokens": 353, "cost": 0, "durationMs": 36433, "outputTokensPerSec": 35.30438329460184, "at": "2026-10-28T18:10:22.178Z" }, "lastUpdated": "2026-10-28T18:10:22.178Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:10:26.057Z" } }, "source": "engine-log" }, "derived": { "durationMs": 455219, "meanTokensPerSec": 31, "genTokensPerSec": 31, "genTokens": 8285, "promptTokensPerSec": null, "peakRssMb": 18256, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 46447, "output": 2255 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 562, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-developer-large-pr-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "large-pr-review", "repetition": 1 }, "trialId": "large-pr-review-mlx-gemma4-12b-q4-2026-10-28T18-10-52-383Z-zc7k", "success": false, "includedInCapability": true, "durationMs": 2861651, "timing": { "startedAt": "2026-10-28T18:10:52.383Z", "finishedAt": "2026-10-28T18:58:34.034Z", "timeToFirstArtifactMs": 277970, "timeToLastArtifactWriteMs": 2805391, "timeToFirstTokenMs": 18376, "firstTurnTtftMs": 16605, "timeToFirstToolCallMs": 19110 }, "metrics": { "process": { "sampleCount": 569, "peakRssMb": 10405, "peakCpuPercent": 161, "firstAtMs": 1, "lastAtMs": 2857815 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 569, "peakUsedMb": 87225, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 175769, "totalOutputTokens": 68492, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 471968, "todayTokensCached": 468347, "todayTokensOut": 8525, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 471968, "totalTokensCached": 468347, "totalTokensOut": 8525, "totalCost": 0, "medianOutputTokensPerSec": 28.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 28.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 175193, "cachedInputTokens": 174985, "outputTokens": 61, "cost": 0, "durationMs": 1110636, "outputTokensPerSec": 23.83548706800052, "at": "2026-10-28T18:57:37.774Z" }, "lastUpdated": "2026-10-28T18:57:37.774Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:58:31.802Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2858359, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 68492, "promptTokensPerSec": null, "peakRssMb": 10405, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 471968, "output": 8525 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 361, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-large-pr-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "large-pr-review", "repetition": 1 }, "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T18-58-56-312Z-v6r4", "success": false, "includedInCapability": true, "durationMs": 827239, "timing": { "startedAt": "2026-10-28T18:58:56.312Z", "finishedAt": "2026-10-28T19:12:43.551Z", "timeToFirstArtifactMs": 38662, "timeToLastArtifactWriteMs": 814311, "timeToFirstTokenMs": 18114, "firstTurnTtftMs": 16159, "timeToFirstToolCallMs": 21522 }, "metrics": { "process": { "sampleCount": 164, "peakRssMb": 30423, "peakCpuPercent": 115.9, "firstAtMs": 0, "lastAtMs": 818403 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 164, "peakUsedMb": 88910, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 182326, "totalOutputTokens": 33507, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 156580, "todayTokensCached": 128908, "todayTokensOut": 5833, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 156580, "totalTokensCached": 128908, "totalTokensOut": 5833, "totalCost": 0, "medianOutputTokensPerSec": 51.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 51.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 31243, "cachedInputTokens": 29349, "outputTokens": 48, "cost": 0, "durationMs": 95031, "outputTokensPerSec": 56.914438224221094, "at": "2026-10-28T19:12:30.623Z" }, "lastUpdated": "2026-10-28T19:12:30.623Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T19:12:36.281Z" } }, "source": "engine-log" }, "derived": { "durationMs": 818931, "meanTokensPerSec": 61.71, "genTokensPerSec": 61.71, "genTokens": 33507, "promptTokensPerSec": 783.14, "peakRssMb": 30423, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 156580, "output": 5833 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-large-pr-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "large-pr-review", "repetition": 1 }, "trialId": "large-pr-review-mlx-qwen3-8-27b-q4-2026-10-28T19-13-05-081Z-wfxq", "success": true, "includedInCapability": true, "durationMs": 2527434, "timing": { "startedAt": "2026-10-28T19:13:05.082Z", "finishedAt": "2026-10-28T19:55:12.516Z", "timeToFirstArtifactMs": 2118017, "timeToLastArtifactWriteMs": 2520309, "timeToFirstTokenMs": 31828, "firstTurnTtftMs": 30028, "timeToFirstToolCallMs": 33943 }, "metrics": { "process": { "sampleCount": 503, "peakRssMb": 18005, "peakCpuPercent": 129.5, "firstAtMs": 0, "lastAtMs": 2520573 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 503, "peakUsedMb": 97432, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 105429, "totalOutputTokens": 46004, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 109369, "todayTokensCached": 109325, "todayTokensOut": 529, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 109369, "totalTokensCached": 109325, "totalTokensOut": 529, "totalCost": 0, "medianOutputTokensPerSec": 16.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 16.7, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 109369, "cachedInputTokens": 109325, "outputTokens": 529, "cost": 0, "durationMs": 2116216, "outputTokensPerSec": 16.674483862090554, "at": "2026-10-28T19:48:23.098Z" }, "lastUpdated": "2026-10-28T19:48:23.098Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T19:55:07.255Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2521104, "meanTokensPerSec": 21, "genTokensPerSec": 21, "genTokens": 46004, "promptTokensPerSec": null, "peakRssMb": 18005, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 109369, "output": 529 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 389, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-large-pr-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "large-pr-review", "repetition": 1 }, "trialId": "large-pr-review-qwen3-8-27b-q4-2026-10-28T19-55-34-508Z-k7yi", "success": true, "includedInCapability": true, "durationMs": 2362214, "timing": { "startedAt": "2026-10-28T19:55:34.509Z", "finishedAt": "2026-10-28T20:34:56.723Z", "timeToFirstArtifactMs": 1341595, "timeToLastArtifactWriteMs": 2354967, "timeToFirstTokenMs": 37403, "firstTurnTtftMs": 35445, "timeToFirstToolCallMs": 39463 }, "metrics": { "process": { "sampleCount": 470, "peakRssMb": 41363, "peakCpuPercent": 118.6, "firstAtMs": 0, "lastAtMs": 2354735 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 470, "peakUsedMb": 103300, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 196408, "totalOutputTokens": 36863, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 203987, "todayTokensCached": 203663, "todayTokensOut": 756, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 203987, "totalTokensCached": 203663, "totalTokensOut": 756, "totalCost": 0, "medianOutputTokensPerSec": 17.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 17.1, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 104704, "cachedInputTokens": 104544, "outputTokens": 165, "cost": 0, "durationMs": 923817, "outputTokensPerSec": 13.856079270291554, "at": "2026-10-28T20:33:19.949Z" }, "lastUpdated": "2026-10-28T20:33:19.949Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T20:34:50.839Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2355280, "meanTokensPerSec": 23.69, "genTokensPerSec": 23.69, "genTokens": 36863, "promptTokensPerSec": 249.23, "peakRssMb": 41363, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 203987, "output": 756 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-developer-craftbook-codemod-sweep", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 1 }, "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T20-35-18-397Z-uz1k", "success": false, "includedInCapability": true, "durationMs": 1152100, "timing": { "startedAt": "2026-10-28T20:35:18.397Z", "finishedAt": "2026-10-28T20:54:30.497Z", "timeToFirstArtifactMs": 39284, "timeToLastArtifactWriteMs": 1119720, "timeToFirstTokenMs": 19164, "firstTurnTtftMs": 12439, "timeToFirstToolCallMs": 25219 }, "metrics": { "process": { "sampleCount": 229, "peakRssMb": 29931, "peakCpuPercent": 102.6, "firstAtMs": 0, "lastAtMs": 1145565 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 229, "peakUsedMb": 96216, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 212229, "totalOutputTokens": 34370, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 27, "todayTokensIn": 514466, "todayTokensCached": 506656, "todayTokensOut": 8665, "todayCost": 0, "totalTurns": 27, "totalTokensIn": 514466, "totalTokensCached": 506656, "totalTokensOut": 8665, "totalCost": 0, "medianOutputTokensPerSec": 66.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 66.8, "turns": 27 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22519, "cachedInputTokens": 22112, "outputTokens": 235, "cost": 0, "durationMs": 32865, "outputTokensPerSec": 66.25291626084396, "at": "2026-10-28T20:53:58.117Z" }, "lastUpdated": "2026-10-28T20:53:58.117Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T20:54:28.323Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1148877, "meanTokensPerSec": 58.75, "genTokensPerSec": 58.75, "genTokens": 34370, "promptTokensPerSec": 821.7, "peakRssMb": 29931, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 514466, "output": 8665 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-craftbook-codemod-sweep", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 1 }, "trialId": "craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-28T20-54-52-653Z-7pru", "success": false, "includedInCapability": true, "durationMs": 1613140, "timing": { "startedAt": "2026-10-28T20:54:52.653Z", "finishedAt": "2026-10-28T21:21:45.793Z", "timeToFirstArtifactMs": 187126, "timeToLastArtifactWriteMs": 1464676, "timeToFirstTokenMs": 23299, "firstTurnTtftMs": 16769, "timeToFirstToolCallMs": 26579 }, "metrics": { "process": { "sampleCount": 321, "peakRssMb": 18241, "peakCpuPercent": 139.5, "firstAtMs": 0, "lastAtMs": 1607021 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 321, "peakUsedMb": 96790, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 56325, "totalOutputTokens": 27298, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 33649, "todayTokensCached": 32743, "todayTokensOut": 4296, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 33649, "totalTokensCached": 32743, "totalTokensOut": 4296, "totalCost": 0, "medianOutputTokensPerSec": 22.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.8, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11484, "cachedInputTokens": 11434, "outputTokens": 1913, "cost": 0, "durationMs": 174036, "outputTokensPerSec": 22.83507565012601, "at": "2026-10-28T21:04:53.444Z" }, "lastUpdated": "2026-10-28T21:04:53.444Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T21:21:43.788Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1610065, "meanTokensPerSec": 20, "genTokensPerSec": 20, "genTokens": 27298, "promptTokensPerSec": null, "peakRssMb": 18241, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 33649, "output": 4296 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 468, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-craftbook-codemod-sweep", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 1 }, "trialId": "craftbook-codemod-sweep-qwen3-8-27b-q4-2026-10-28T21-22-07-237Z-xpfj", "success": false, "includedInCapability": true, "durationMs": 5784854, "timing": { "startedAt": "2026-10-28T21:22:07.237Z", "finishedAt": "2026-10-28T22:58:32.091Z", "timeToFirstArtifactMs": 230852, "timeToLastArtifactWriteMs": 5773656, "timeToFirstTokenMs": 33997, "firstTurnTtftMs": 27295, "timeToFirstToolCallMs": 40459 }, "metrics": { "process": { "sampleCount": 1150, "peakRssMb": 47480, "peakCpuPercent": 124.7, "firstAtMs": 0, "lastAtMs": 5769074 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1150, "peakUsedMb": 109895, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 256338, "totalOutputTokens": 80778, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 173274, "todayTokensCached": 155505, "todayTokensOut": 15580, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 173274, "totalTokensCached": 155505, "totalTokensOut": 15580, "totalCost": 0, "medianOutputTokensPerSec": 20.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 20.5, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 29906, "cachedInputTokens": 29624, "outputTokens": 1271, "cost": 0, "durationMs": 230059, "outputTokensPerSec": 21.624259790519833, "at": "2026-10-28T22:30:09.402Z" }, "lastUpdated": "2026-10-28T22:30:09.402Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T22:58:21.928Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5773642, "meanTokensPerSec": 19, "genTokensPerSec": 19, "genTokens": 80778, "promptTokensPerSec": 322.34, "peakRssMb": 47480, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 173274, "output": 15580 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-craftbook-codemod-sweep", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 1 }, "trialId": "craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-28T22-58-54-564Z-dp8j", "success": false, "includedInCapability": true, "durationMs": 1656027, "timing": { "startedAt": "2026-10-28T22:58:54.564Z", "finishedAt": "2026-10-28T23:26:30.591Z", "timeToFirstArtifactMs": 56242, "timeToLastArtifactWriteMs": 1393451, "timeToFirstTokenMs": 21667, "firstTurnTtftMs": 15147, "timeToFirstToolCallMs": 22077 }, "metrics": { "process": { "sampleCount": 329, "peakRssMb": 11162, "peakCpuPercent": 165.3, "firstAtMs": 0, "lastAtMs": 1650156 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 329, "peakUsedMb": 82220, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 161610, "totalOutputTokens": 38216, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 131901, "todayTokensCached": 127892, "todayTokensOut": 4987, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 131901, "totalTokensCached": 127892, "totalTokensOut": 4987, "totalCost": 0, "medianOutputTokensPerSec": 34.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.3, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 29028, "cachedInputTokens": 28617, "outputTokens": 998, "cost": 0, "durationMs": 34664, "outputTokensPerSec": 29.614725124457877, "at": "2026-10-28T23:20:21.532Z" }, "lastUpdated": "2026-10-28T23:20:21.532Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T23:26:28.345Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1652720, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 38216, "promptTokensPerSec": null, "peakRssMb": 11162, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 131901, "output": 4987 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 818.5, "sampleCount": 18, "structuredSampleCount": 0, "firstTokenSampleCount": 18, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-developer-craftbook-refactor-module", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 1 }, "trialId": "craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-28T23-26-52-344Z-bssk", "success": false, "includedInCapability": true, "durationMs": 2209689, "timing": { "startedAt": "2026-10-28T23:26:52.344Z", "finishedAt": "2026-10-29T00:03:42.033Z", "timeToFirstArtifactMs": 40472, "timeToLastArtifactWriteMs": 2205192, "timeToFirstTokenMs": 22584, "firstTurnTtftMs": 16057, "timeToFirstToolCallMs": 40471 }, "metrics": { "process": { "sampleCount": 439, "peakRssMb": 18660, "peakCpuPercent": 110.6, "firstAtMs": 0, "lastAtMs": 2201832 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 439, "peakUsedMb": 97264, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 280510, "totalOutputTokens": 21514, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 77039, "todayTokensCached": 7740, "todayTokensOut": 708, "todayCost": 0, "totalTurns": 21, "totalTokensIn": 390862, "totalTokensCached": 201906, "totalTokensOut": 17479, "totalCost": 0, "medianOutputTokensPerSec": 31.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.3, "turns": 21 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 27161, "cachedInputTokens": 3200, "outputTokens": 215, "cost": 0, "durationMs": 70359, "outputTokensPerSec": 16.45369674588565, "at": "2026-10-29T00:03:37.538Z" }, "lastUpdated": "2026-10-29T00:03:37.538Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T00:03:39.304Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2205895, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 21514, "promptTokensPerSec": null, "peakRssMb": 18660, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 390862, "output": 17479 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 485.5, "sampleCount": 22, "structuredSampleCount": 0, "firstTokenSampleCount": 22, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-developer-craftbook-refactor-module", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 1 }, "trialId": "craftbook-refactor-module-qwen3-8-27b-q4-2026-10-29T00-04-05-657Z-org4", "success": false, "includedInCapability": true, "durationMs": 2459504, "timing": { "startedAt": "2026-10-29T00:04:05.657Z", "finishedAt": "2026-10-29T00:45:05.161Z", "timeToFirstArtifactMs": 67821, "timeToLastArtifactWriteMs": 2353314, "timeToFirstTokenMs": 28876, "firstTurnTtftMs": 22162, "timeToFirstToolCallMs": 67820 }, "metrics": { "process": { "sampleCount": 489, "peakRssMb": 49844, "peakCpuPercent": 118.7, "firstAtMs": 0, "lastAtMs": 2450586 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 489, "peakUsedMb": 112357, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 218302, "totalOutputTokens": 41393, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 223023, "todayTokensCached": 126160, "todayTokensOut": 12374, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 223023, "totalTokensCached": 126160, "totalTokensOut": 12374, "totalCost": 0, "medianOutputTokensPerSec": 30.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.7, "turns": 14 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 32916, "cachedInputTokens": 32670, "outputTokens": 1250, "cost": 0, "durationMs": 61240, "outputTokensPerSec": 32.34343769258135, "at": "2026-10-29T00:34:11.641Z" }, "lastUpdated": "2026-10-29T00:34:11.641Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T00:44:59.602Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2452899, "meanTokensPerSec": 25.37, "genTokensPerSec": 25.37, "genTokens": 41393, "promptTokensPerSec": 322.29, "peakRssMb": 49844, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 223023, "output": 12374 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-craftbook-refactor-module", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 1 }, "trialId": "craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-29T00-45-48-041Z-7jsj", "success": false, "includedInCapability": true, "durationMs": 1256944, "timing": { "startedAt": "2026-10-29T00:45:48.042Z", "finishedAt": "2026-10-29T01:06:44.986Z", "timeToFirstArtifactMs": 27964, "timeToLastArtifactWriteMs": 1254699, "timeToFirstTokenMs": 18448, "firstTurnTtftMs": 11910, "timeToFirstToolCallMs": 27963 }, "metrics": { "process": { "sampleCount": 250, "peakRssMb": 11719, "peakCpuPercent": 120, "firstAtMs": 0, "lastAtMs": 1252192 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 250, "peakUsedMb": 80257, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 204709, "totalOutputTokens": 12271, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 23, "todayTokensIn": 449928, "todayTokensCached": 312704, "todayTokensOut": 8656, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 449928, "totalTokensCached": 312704, "totalTokensOut": 8656, "totalCost": 0, "medianOutputTokensPerSec": 40.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.5, "turns": 23 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 29339, "cachedInputTokens": 28472, "outputTokens": 192, "cost": 0, "durationMs": 4807, "outputTokensPerSec": 48.234254832548395, "at": "2026-10-29T01:06:42.741Z" }, "lastUpdated": "2026-10-29T01:06:42.741Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T01:06:43.317Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1254223, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 12271, "promptTokensPerSec": null, "peakRssMb": 11719, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 449928, "output": 8656 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1101.5, "sampleCount": 24, "structuredSampleCount": 0, "firstTokenSampleCount": 24, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-craftbook-refactor-module", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 1 }, "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-29T01-07-25-131Z-x3gz", "success": false, "includedInCapability": true, "durationMs": 1806730, "timing": { "startedAt": "2026-10-29T01:07:25.131Z", "finishedAt": "2026-10-29T01:37:31.861Z", "timeToFirstArtifactMs": 17721, "timeToLastArtifactWriteMs": 1796826, "timeToFirstTokenMs": 15174, "firstTurnTtftMs": 8247, "timeToFirstToolCallMs": 17723 }, "metrics": { "process": { "sampleCount": 359, "peakRssMb": 30036, "peakCpuPercent": 146, "firstAtMs": 0, "lastAtMs": 1796977 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 359, "peakUsedMb": 82162, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 728504, "totalOutputTokens": 36056, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 32, "todayTokensIn": 663635, "todayTokensCached": 414623, "todayTokensOut": 15618, "todayCost": 0, "totalTurns": 32, "totalTokensIn": 663635, "totalTokensCached": 414623, "totalTokensOut": 15618, "totalCost": 0, "medianOutputTokensPerSec": 65, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 65, "turns": 32 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 32576, "outputTokens": 535, "cost": 0, "durationMs": 61186, "outputTokensPerSec": 61.726425707045, "at": "2026-10-29T01:30:50.050Z" }, "lastUpdated": "2026-10-29T01:30:50.050Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T01:37:25.156Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1798782, "meanTokensPerSec": 55.84, "genTokensPerSec": 55.84, "genTokens": 36056, "promptTokensPerSec": 764.02, "peakRssMb": 30036, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 663635, "output": 15618 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-developer-craftbook-bug-fix-tdd", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 1 }, "trialId": "craftbook-bug-fix-tdd-qwen3-8-27b-q4-2026-10-29T01-38-17-330Z-6knk", "success": false, "includedInCapability": true, "durationMs": 3641768, "timing": { "startedAt": "2026-10-29T01:38:17.330Z", "finishedAt": "2026-10-29T02:38:59.098Z", "timeToFirstArtifactMs": 137885, "timeToLastArtifactWriteMs": 3391770, "timeToFirstTokenMs": 33111, "firstTurnTtftMs": 26490, "timeToFirstToolCallMs": 104839 }, "metrics": { "process": { "sampleCount": 725, "peakRssMb": 48901, "peakCpuPercent": 174.9, "firstAtMs": 0, "lastAtMs": 3633039 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 725, "peakUsedMb": 103315, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 285253, "totalOutputTokens": 57279, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 228075, "todayTokensCached": 180893, "todayTokensOut": 14067, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 228075, "totalTokensCached": 180893, "totalTokensOut": 14067, "totalCost": 0, "medianOutputTokensPerSec": 21.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 21.5, "turns": 14 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 29220, "cachedInputTokens": 28432, "outputTokens": 603, "cost": 0, "durationMs": 168661, "outputTokensPerSec": 32.88181108208797, "at": "2026-10-29T02:34:49.100Z" }, "lastUpdated": "2026-10-29T02:34:49.100Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T02:38:52.937Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3634569, "meanTokensPerSec": 20.86, "genTokensPerSec": 20.86, "genTokens": 57279, "promptTokensPerSec": 347.32, "peakRssMb": 48901, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 228075, "output": 14067 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-developer-craftbook-bug-fix-tdd", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 1 }, "trialId": "craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-29T02-39-20-666Z-fe1m", "success": false, "includedInCapability": true, "durationMs": 2233652, "timing": { "startedAt": "2026-10-29T02:39:20.667Z", "finishedAt": "2026-10-29T03:16:34.319Z", "timeToFirstArtifactMs": 32435, "timeToLastArtifactWriteMs": 2228343, "timeToFirstTokenMs": 21848, "firstTurnTtftMs": 15311, "timeToFirstToolCallMs": 22420 }, "metrics": { "process": { "sampleCount": 444, "peakRssMb": 11868, "peakCpuPercent": 161.2, "firstAtMs": 0, "lastAtMs": 2225391 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 444, "peakUsedMb": 73833, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 773311, "totalOutputTokens": 20708, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 30, "todayTokensIn": 735963, "todayTokensCached": 441412, "todayTokensOut": 12177, "todayCost": 0, "totalTurns": 30, "totalTokensIn": 735963, "totalTokensCached": 441412, "totalTokensOut": 12177, "totalCost": 0, "medianOutputTokensPerSec": 33.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 33.8, "turns": 30 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 46245, "cachedInputTokens": 1613, "outputTokens": 161, "cost": 0, "durationMs": 43984, "outputTokensPerSec": 45.09139215162308, "at": "2026-10-29T03:16:29.010Z" }, "lastUpdated": "2026-10-29T03:16:29.010Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T03:16:31.919Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2230194, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 20708, "promptTokensPerSec": null, "peakRssMb": 11868, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 735963, "output": 12177 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 906.5, "sampleCount": 42, "structuredSampleCount": 0, "firstTokenSampleCount": 42, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-developer-craftbook-bug-fix-tdd", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 1 }, "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-29T03-16-58-124Z-mv1g", "success": false, "includedInCapability": true, "durationMs": 1928584, "timing": { "startedAt": "2026-10-29T03:16:58.125Z", "finishedAt": "2026-10-29T03:49:06.709Z", "timeToFirstArtifactMs": 26798, "timeToLastArtifactWriteMs": 1907682, "timeToFirstTokenMs": 19242, "firstTurnTtftMs": 12477, "timeToFirstToolCallMs": 22272 }, "metrics": { "process": { "sampleCount": 384, "peakRssMb": 30456, "peakCpuPercent": 128.3, "firstAtMs": 0, "lastAtMs": 1922422 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 384, "peakUsedMb": 85158, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 916434, "totalOutputTokens": 22182, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 20, "todayTokensIn": 498138, "todayTokensCached": 200301, "todayTokensOut": 6450, "todayCost": 0, "totalTurns": 20, "totalTokensIn": 498138, "totalTokensCached": 200301, "totalTokensOut": 6450, "totalCost": 0, "medianOutputTokensPerSec": 71.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 71.7, "turns": 20 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 62896, "outputTokens": 344, "cost": 0, "durationMs": 111162, "outputTokensPerSec": 40.13963913411192, "at": "2026-10-29T03:48:45.807Z" }, "lastUpdated": "2026-10-29T03:48:45.807Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T03:49:02.622Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1923450, "meanTokensPerSec": 53.86, "genTokensPerSec": 53.86, "genTokens": 22182, "promptTokensPerSec": 719.7, "peakRssMb": 30456, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 498138, "output": 6450 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-developer-craftbook-bug-fix-tdd", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 1 }, "trialId": "craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-29T03-49-52-142Z-n6ra", "success": false, "includedInCapability": true, "durationMs": 2502351, "timing": { "startedAt": "2026-10-29T03:49:52.142Z", "finishedAt": "2026-10-29T04:31:34.493Z", "timeToFirstArtifactMs": 228601, "timeToLastArtifactWriteMs": 2495854, "timeToFirstTokenMs": 28306, "firstTurnTtftMs": 21782, "timeToFirstToolCallMs": 92132 }, "metrics": { "process": { "sampleCount": 498, "peakRssMb": 18745, "peakCpuPercent": 118.5, "firstAtMs": 0, "lastAtMs": 2496824 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 498, "peakUsedMb": 90188, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 295087, "totalOutputTokens": 23325, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 354826, "todayTokensCached": 225113, "todayTokensOut": 16471, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 354826, "totalTokensCached": 225113, "totalTokensOut": 16471, "totalCost": 0, "medianOutputTokensPerSec": 32.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.1, "turns": 22 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 25502, "cachedInputTokens": 25402, "outputTokens": 768, "cost": 0, "durationMs": 236660, "outputTokensPerSec": 20.982930396248324, "at": "2026-10-29T04:31:27.996Z" }, "lastUpdated": "2026-10-29T04:31:27.996Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T04:31:31.812Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2498617, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 23325, "promptTokensPerSec": null, "peakRssMb": 18745, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 354826, "output": 16471 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 502, "sampleCount": 23, "structuredSampleCount": 0, "firstTokenSampleCount": 23, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-find-vs-create", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 1 }, "trialId": "craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-29T04-31-56-174Z-jchw", "success": true, "includedInCapability": true, "durationMs": 429523, "timing": { "startedAt": "2026-10-29T04:31:56.175Z", "finishedAt": "2026-10-29T04:39:05.698Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 18587, "firstTurnTtftMs": 16877, "timeToFirstToolCallMs": 59038 }, "metrics": { "process": { "sampleCount": 85, "peakRssMb": 11190, "peakCpuPercent": 144, "firstAtMs": 0, "lastAtMs": 421778 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 85, "peakUsedMb": 74960, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 33643, "totalOutputTokens": 12546, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 20930, "todayTokensCached": 20516, "todayTokensOut": 8211, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 20930, "totalTokensCached": 20516, "totalTokensOut": 8211, "totalCost": 0, "medianOutputTokensPerSec": 36.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.7, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10597, "cachedInputTokens": 10425, "outputTokens": 19, "cost": 0, "durationMs": 58022, "outputTokensPerSec": 37.96368762275922, "at": "2026-10-29T04:37:45.115Z" }, "lastUpdated": "2026-10-29T04:37:45.115Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T04:39:00.807Z" } }, "source": "engine-log" }, "derived": { "durationMs": 423563, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 12546, "promptTokensPerSec": null, "peakRssMb": 11190, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 20930, "output": 8211 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 982, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-find-vs-create", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 1 }, "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-29T04-39-28-341Z-ii9j", "success": false, "includedInCapability": true, "durationMs": 447195, "timing": { "startedAt": "2026-10-29T04:39:28.342Z", "finishedAt": "2026-10-29T04:46:55.537Z", "timeToFirstArtifactMs": 151432, "timeToLastArtifactWriteMs": 403484, "timeToFirstTokenMs": 15913, "firstTurnTtftMs": 13978, "timeToFirstToolCallMs": 33823 }, "metrics": { "process": { "sampleCount": 87, "peakRssMb": 28326, "peakCpuPercent": 101.1, "firstAtMs": 0, "lastAtMs": 431574 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 87, "peakUsedMb": 82013, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 145497, "totalOutputTokens": 13858, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 147082, "todayTokensCached": 54276, "todayTokensOut": 4739, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 147082, "totalTokensCached": 54276, "totalTokensOut": 4739, "totalCost": 0, "medianOutputTokensPerSec": 53.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 53.4, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18405, "cachedInputTokens": 18240, "outputTokens": 1189, "cost": 0, "durationMs": 29508, "outputTokensPerSec": 41.0587205369983, "at": "2026-10-29T04:46:41.343Z" }, "lastUpdated": "2026-10-29T04:46:41.343Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T04:46:42.500Z" } }, "source": "engine-log" }, "derived": { "durationMs": 433109, "meanTokensPerSec": 56.91, "genTokensPerSec": 56.91, "genTokens": 13858, "promptTokensPerSec": 878.89, "peakRssMb": 28326, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 147082, "output": 4739 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-find-vs-create", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 1 }, "trialId": "craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-29T04-47-18-111Z-54nx", "success": false, "includedInCapability": true, "durationMs": 1064822, "timing": { "startedAt": "2026-10-29T04:47:18.112Z", "finishedAt": "2026-10-29T05:05:02.934Z", "timeToFirstArtifactMs": 442135, "timeToLastArtifactWriteMs": 1055295, "timeToFirstTokenMs": 30887, "firstTurnTtftMs": 29277, "timeToFirstToolCallMs": 37820 }, "metrics": { "process": { "sampleCount": 211, "peakRssMb": 18542, "peakCpuPercent": 126.6, "firstAtMs": 0, "lastAtMs": 1055103 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 211, "peakUsedMb": 91267, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 79802, "totalOutputTokens": 12823, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 84771, "todayTokensCached": 60016, "todayTokensOut": 2521, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 84771, "totalTokensCached": 60016, "totalTokensOut": 2521, "totalCost": 0, "medianOutputTokensPerSec": 15.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 15.6, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13608, "cachedInputTokens": 2272, "outputTokens": 181, "cost": 0, "durationMs": 61742, "outputTokensPerSec": 15.354974651278965, "at": "2026-10-29T05:04:53.412Z" }, "lastUpdated": "2026-10-29T05:04:53.412Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T05:04:57.496Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1058420, "meanTokensPerSec": 14, "genTokensPerSec": 14, "genTokens": 12823, "promptTokensPerSec": null, "peakRssMb": 18542, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 84771, "output": 2521 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 502, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-find-vs-create", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 1 }, "trialId": "craftbook-find-vs-create-qwen3-8-27b-q4-2026-10-29T05-05-25-371Z-tnv7", "success": true, "includedInCapability": true, "durationMs": 86917, "timing": { "startedAt": "2026-10-29T05:05:25.372Z", "finishedAt": "2026-10-29T05:06:52.289Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 35922, "firstTurnTtftMs": 34029, "timeToFirstToolCallMs": 49009 }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 37580, "peakCpuPercent": 110.5, "firstAtMs": 0, "lastAtMs": 80285 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 93594, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13427, "totalOutputTokens": 1077, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T05:06:47.486Z" } }, "source": "engine-log" }, "derived": { "durationMs": 81064, "meanTokensPerSec": 27.98, "genTokensPerSec": 27.98, "genTokens": 1077, "promptTokensPerSec": 414.7, "peakRssMb": 37580, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-edit-midtask", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 1 }, "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-29T05-07-15-525Z-zid9", "success": true, "includedInCapability": true, "durationMs": 195932, "timing": { "startedAt": "2026-10-29T05:07:15.526Z", "finishedAt": "2026-10-29T05:10:31.458Z", "timeToFirstArtifactMs": 106743, "timeToLastArtifactWriteMs": 152270, "timeToFirstTokenMs": 18728, "firstTurnTtftMs": 16823, "timeToFirstToolCallMs": 24074 }, "metrics": { "process": { "sampleCount": 38, "peakRssMb": 26645, "peakCpuPercent": 116.4, "firstAtMs": 0, "lastAtMs": 185664 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 38, "peakUsedMb": 87932, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 59511, "totalOutputTokens": 5982, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 134580, "todayTokensCached": 126791, "todayTokensOut": 1183, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 134580, "totalTokensCached": 126791, "totalTokensOut": 1183, "totalCost": 0, "medianOutputTokensPerSec": 54.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 54.6, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22925, "cachedInputTokens": 21559, "outputTokens": 30, "cost": 0, "durationMs": 22843, "outputTokensPerSec": 85.23019500080822, "at": "2026-10-29T05:10:21.454Z" }, "lastUpdated": "2026-10-29T05:10:21.454Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T05:10:22.787Z" } }, "source": "engine-log" }, "derived": { "durationMs": 186213, "meanTokensPerSec": 59.39, "genTokensPerSec": 59.39, "genTokens": 5982, "promptTokensPerSec": 923.57, "peakRssMb": 26645, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 134580, "output": 1183 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-edit-midtask", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 1 }, "trialId": "craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-29T05-10-53-786Z-0wam", "success": true, "includedInCapability": true, "durationMs": 1320112, "timing": { "startedAt": "2026-10-29T05:10:53.787Z", "finishedAt": "2026-10-29T05:32:53.899Z", "timeToFirstArtifactMs": 966488, "timeToLastArtifactWriteMs": 1288063, "timeToFirstTokenMs": 42068, "firstTurnTtftMs": 40373, "timeToFirstToolCallMs": 49741 }, "metrics": { "process": { "sampleCount": 263, "peakRssMb": 18920, "peakCpuPercent": 123.1, "firstAtMs": 0, "lastAtMs": 1314989 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 263, "peakUsedMb": 92779, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 74385, "totalOutputTokens": 22795, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 205975, "todayTokensCached": 191741, "todayTokensOut": 6155, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 205975, "totalTokensCached": 191741, "totalTokensOut": 6155, "totalCost": 0, "medianOutputTokensPerSec": 21.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 21.9, "turns": 11 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 29336, "cachedInputTokens": 29162, "outputTokens": 298, "cost": 0, "durationMs": 24873, "outputTokensPerSec": 22.372290309432664, "at": "2026-10-29T05:32:46.727Z" }, "lastUpdated": "2026-10-29T05:32:46.727Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T05:32:50.373Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1315516, "meanTokensPerSec": 21, "genTokensPerSec": 21, "genTokens": 22795, "promptTokensPerSec": null, "peakRssMb": 18920, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 205975, "output": 6155 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 363.5, "sampleCount": 12, "structuredSampleCount": 0, "firstTokenSampleCount": 12, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-edit-midtask", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 1 }, "trialId": "craftbook-edit-midtask-qwen3-8-27b-q4-2026-10-29T05-33-16-492Z-zwbq", "success": true, "includedInCapability": true, "durationMs": 1408530, "timing": { "startedAt": "2026-10-29T05:33:16.492Z", "finishedAt": "2026-10-29T05:56:45.022Z", "timeToFirstArtifactMs": 985668, "timeToLastArtifactWriteMs": 1388225, "timeToFirstTokenMs": 51747, "firstTurnTtftMs": 49893, "timeToFirstToolCallMs": 60824 }, "metrics": { "process": { "sampleCount": 281, "peakRssMb": 46849, "peakCpuPercent": 100, "firstAtMs": 0, "lastAtMs": 1405053 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 281, "peakUsedMb": 111102, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 71042, "totalOutputTokens": 24517, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 146698, "todayTokensCached": 137551, "todayTokensOut": 2768, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 146698, "totalTokensCached": 137551, "totalTokensOut": 2768, "totalCost": 0, "medianOutputTokensPerSec": 22.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.6, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 32564, "cachedInputTokens": 30529, "outputTokens": 128, "cost": 0, "durationMs": 27055, "outputTokensPerSec": 26.71728505260465, "at": "2026-10-29T05:56:38.482Z" }, "lastUpdated": "2026-10-29T05:56:38.482Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T05:56:43.122Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1405578, "meanTokensPerSec": 21.09, "genTokensPerSec": 21.09, "genTokens": 24517, "promptTokensPerSec": 340.63, "peakRssMb": 46849, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 146698, "output": 2768 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-edit-midtask", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 1 }, "trialId": "craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-29T05-57-06-675Z-sqvj", "success": true, "includedInCapability": true, "durationMs": 475323, "timing": { "startedAt": "2026-10-29T05:57:06.676Z", "finishedAt": "2026-10-29T06:05:01.999Z", "timeToFirstArtifactMs": 146734, "timeToLastArtifactWriteMs": 443718, "timeToFirstTokenMs": 22146, "firstTurnTtftMs": 20459, "timeToFirstToolCallMs": 33965 }, "metrics": { "process": { "sampleCount": 95, "peakRssMb": 11265, "peakCpuPercent": 100.2, "firstAtMs": 0, "lastAtMs": 471973 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 95, "peakUsedMb": 74672, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44471, "totalOutputTokens": 11488, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 164242, "todayTokensCached": 158389, "todayTokensOut": 3411, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 164242, "totalTokensCached": 158389, "totalTokensOut": 3411, "totalCost": 0, "medianOutputTokensPerSec": 35.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.7, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10021, "cachedInputTokens": 9642, "outputTokens": 32, "cost": 0, "durationMs": 22097, "outputTokensPerSec": 38.62689488780641, "at": "2026-10-29T06:04:55.438Z" }, "lastUpdated": "2026-10-29T06:04:55.438Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T06:05:00.265Z" } }, "source": "engine-log" }, "derived": { "durationMs": 472523, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 11488, "promptTokensPerSec": null, "peakRssMb": 11265, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 164242, "output": 3411 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 736, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-invoice-run", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 1 }, "trialId": "craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-29T06-05-23-199Z-bgk3", "success": true, "includedInCapability": true, "durationMs": 690415, "timing": { "startedAt": "2026-10-29T06:05:23.200Z", "finishedAt": "2026-10-29T06:16:53.615Z", "timeToFirstArtifactMs": 74975, "timeToLastArtifactWriteMs": 671611, "timeToFirstTokenMs": 27231, "firstTurnTtftMs": 20727, "timeToFirstToolCallMs": 28476 }, "metrics": { "process": { "sampleCount": 135, "peakRssMb": 19598, "peakCpuPercent": 146.8, "firstAtMs": 0, "lastAtMs": 672481 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 135, "peakUsedMb": 90918, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 84731, "totalOutputTokens": 11460, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 84283, "todayTokensCached": 83268, "todayTokensOut": 1081, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 84283, "totalTokensCached": 83268, "totalTokensOut": 1081, "totalCost": 0, "medianOutputTokensPerSec": 32.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.4, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 22209, "cachedInputTokens": 21711, "outputTokens": 469, "cost": 0, "durationMs": 209934, "outputTokensPerSec": 23.5583123000469, "at": "2026-10-29T06:16:34.811Z" }, "lastUpdated": "2026-10-29T06:16:34.811Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T06:16:37.296Z" } }, "source": "engine-log" }, "derived": { "durationMs": 673028, "meanTokensPerSec": 31, "genTokensPerSec": 31, "genTokens": 11460, "promptTokensPerSec": null, "peakRssMb": 19598, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 84283, "output": 1081 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 561, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-invoice-run", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 1 }, "trialId": "craftbook-invoice-run-qwen3-8-27b-q4-2026-10-29T06-17-15-098Z-lnw2", "success": true, "includedInCapability": true, "durationMs": 725750, "timing": { "startedAt": "2026-10-29T06:17:15.099Z", "finishedAt": "2026-10-29T06:29:20.849Z", "timeToFirstArtifactMs": 77634, "timeToLastArtifactWriteMs": 714370, "timeToFirstTokenMs": 32152, "firstTurnTtftMs": 25464, "timeToFirstToolCallMs": 33736 }, "metrics": { "process": { "sampleCount": 144, "peakRssMb": 49186, "peakCpuPercent": 130.5, "firstAtMs": 0, "lastAtMs": 717594 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 144, "peakUsedMb": 108433, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 92126, "totalOutputTokens": 12989, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 79012, "todayTokensCached": 77843, "todayTokensOut": 1193, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 79012, "totalTokensCached": 77843, "totalTokensOut": 1193, "totalCost": 0, "medianOutputTokensPerSec": 38.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 38.8, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 18353, "cachedInputTokens": 17910, "outputTokens": 719, "cost": 0, "durationMs": 242640, "outputTokensPerSec": 26.33681819323925, "at": "2026-10-29T06:29:09.469Z" }, "lastUpdated": "2026-10-29T06:29:09.469Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T06:29:14.272Z" } }, "source": "engine-log" }, "derived": { "durationMs": 718120, "meanTokensPerSec": 30.37, "genTokensPerSec": 30.37, "genTokens": 12989, "promptTokensPerSec": 384.61, "peakRssMb": 49186, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 79012, "output": 1193 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-invoice-run", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 1 }, "trialId": "craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-29T06-29-44-397Z-1hlf", "success": false, "includedInCapability": true, "durationMs": 1322874, "timing": { "startedAt": "2026-10-29T06:29:44.398Z", "finishedAt": "2026-10-29T06:51:47.272Z", "timeToFirstArtifactMs": 38858, "timeToLastArtifactWriteMs": 1307822, "timeToFirstTokenMs": 20680, "firstTurnTtftMs": 14164, "timeToFirstToolCallMs": 26865 }, "metrics": { "process": { "sampleCount": 262, "peakRssMb": 12944, "peakCpuPercent": 125.4, "firstAtMs": 0, "lastAtMs": 1310652 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 262, "peakUsedMb": 75278, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 211144, "totalOutputTokens": 33581, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 305711, "todayTokensCached": 235405, "todayTokensOut": 11948, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 305711, "totalTokensCached": 235405, "totalTokensOut": 11948, "totalCost": 0, "medianOutputTokensPerSec": 35.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.5, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 37518, "cachedInputTokens": 796, "outputTokens": 269, "cost": 0, "durationMs": 50852, "outputTokensPerSec": 32.90032929745761, "at": "2026-10-29T06:51:32.219Z" }, "lastUpdated": "2026-10-29T06:51:32.219Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T06:51:36.893Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1311434, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 33581, "promptTokensPerSec": null, "peakRssMb": 12944, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 305711, "output": 11948 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 953, "sampleCount": 15, "structuredSampleCount": 0, "firstTokenSampleCount": 15, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-invoice-run", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 1 }, "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-29T06-52-12-844Z-o38h", "success": true, "includedInCapability": true, "durationMs": 275796, "timing": { "startedAt": "2026-10-29T06:52:12.845Z", "finishedAt": "2026-10-29T06:56:48.641Z", "timeToFirstArtifactMs": 27838, "timeToLastArtifactWriteMs": 261193, "timeToFirstTokenMs": 18844, "firstTurnTtftMs": 12020, "timeToFirstToolCallMs": 22098 }, "metrics": { "process": { "sampleCount": 53, "peakRssMb": 27939, "peakCpuPercent": 146.6, "firstAtMs": 0, "lastAtMs": 260937 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 53, "peakUsedMb": 89593, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 90485, "totalOutputTokens": 9992, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 75053, "todayTokensCached": 69537, "todayTokensOut": 781, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 75053, "totalTokensCached": 69537, "totalTokensOut": 781, "totalCost": 0, "medianOutputTokensPerSec": 97.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 97.7, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17371, "cachedInputTokens": 17068, "outputTokens": 382, "cost": 0, "durationMs": 54523, "outputTokensPerSec": 86.30447630285215, "at": "2026-10-29T06:56:34.038Z" }, "lastUpdated": "2026-10-29T06:56:34.038Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T06:56:35.482Z" } }, "source": "engine-log" }, "derived": { "durationMs": 261482, "meanTokensPerSec": 81.86, "genTokensPerSec": 81.86, "genTokens": 9992, "promptTokensPerSec": 969.42, "peakRssMb": 27939, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 75053, "output": 781 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-author-linear", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 1 }, "trialId": "craftbook-author-linear-qwen3-8-27b-q4-2026-10-29T06-57-11-271Z-8zyl", "success": true, "includedInCapability": true, "durationMs": 2151618, "timing": { "startedAt": "2026-10-29T06:57:11.271Z", "finishedAt": "2026-10-29T07:33:02.889Z", "timeToFirstArtifactMs": 1048527, "timeToLastArtifactWriteMs": 2145065, "timeToFirstTokenMs": 49062, "firstTurnTtftMs": 47192, "timeToFirstToolCallMs": 57857 }, "metrics": { "process": { "sampleCount": 428, "peakRssMb": 51858, "peakCpuPercent": 102.6, "firstAtMs": 0, "lastAtMs": 2142658 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 428, "peakUsedMb": 114822, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 306456, "totalOutputTokens": 26950, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 205682, "todayTokensCached": 186375, "todayTokensOut": 2181, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 205682, "totalTokensCached": 186375, "totalTokensOut": 2181, "totalCost": 0, "medianOutputTokensPerSec": 25.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.6, "turns": 10 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16466, "cachedInputTokens": 15338, "outputTokens": 82, "cost": 0, "durationMs": 578059, "outputTokensPerSec": 35.992753458970256, "at": "2026-10-29T07:32:56.335Z" }, "lastUpdated": "2026-10-29T07:32:56.335Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T07:32:59.785Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2147462, "meanTokensPerSec": 22.46, "genTokensPerSec": 22.46, "genTokens": 26950, "promptTokensPerSec": 349.09, "peakRssMb": 51858, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 205682, "output": 2181 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-author-linear", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 1 }, "trialId": "craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-29T07-33-23-937Z-njbf", "success": false, "includedInCapability": true, "durationMs": 1041473, "timing": { "startedAt": "2026-10-29T07:33:23.937Z", "finishedAt": "2026-10-29T07:50:45.410Z", "timeToFirstArtifactMs": 138141, "timeToLastArtifactWriteMs": 1027675, "timeToFirstTokenMs": 21399, "firstTurnTtftMs": 19696, "timeToFirstToolCallMs": 51278 }, "metrics": { "process": { "sampleCount": 207, "peakRssMb": 13299, "peakCpuPercent": 102.1, "firstAtMs": 0, "lastAtMs": 1034746 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 207, "peakUsedMb": 75383, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 110258, "totalOutputTokens": 21460, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 400712, "todayTokensCached": 382407, "todayTokensOut": 6279, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 400712, "totalTokensCached": 382407, "totalTokensOut": 6279, "totalCost": 0, "medianOutputTokensPerSec": 37.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.4, "turns": 22 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9421, "cachedInputTokens": 9233, "outputTokens": 52, "cost": 0, "durationMs": 37658, "outputTokensPerSec": 42.49164182216637, "at": "2026-10-29T07:50:31.612Z" }, "lastUpdated": "2026-10-29T07:50:31.612Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T07:50:43.564Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1038567, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 21460, "promptTokensPerSec": null, "peakRssMb": 13299, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 400712, "output": 6279 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 606.5, "sampleCount": 24, "structuredSampleCount": 0, "firstTokenSampleCount": 24, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-author-linear", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 1 }, "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-29T07-51-07-996Z-c9ub", "success": false, "includedInCapability": true, "durationMs": 511791, "timing": { "startedAt": "2026-10-29T07:51:07.997Z", "finishedAt": "2026-10-29T07:59:39.788Z", "timeToFirstArtifactMs": 171754, "timeToLastArtifactWriteMs": 508055, "timeToFirstTokenMs": 19400, "firstTurnTtftMs": 17494, "timeToFirstToolCallMs": 30675 }, "metrics": { "process": { "sampleCount": 102, "peakRssMb": 24299, "peakCpuPercent": 98.2, "firstAtMs": 0, "lastAtMs": 506882 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 102, "peakUsedMb": 83142, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 116500, "totalOutputTokens": 18072, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 227786, "todayTokensCached": 182474, "todayTokensOut": 3492, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 227786, "totalTokensCached": 182474, "totalTokensOut": 3492, "totalCost": 0, "medianOutputTokensPerSec": 58.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 58.8, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10766, "cachedInputTokens": 10713, "outputTokens": 325, "cost": 0, "durationMs": 23734, "outputTokensPerSec": 79.10266329389837, "at": "2026-10-29T07:59:36.052Z" }, "lastUpdated": "2026-10-29T07:59:36.052Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T07:59:37.968Z" } }, "source": "engine-log" }, "derived": { "durationMs": 508923, "meanTokensPerSec": 54.55, "genTokensPerSec": 54.55, "genTokens": 18072, "promptTokensPerSec": 824.36, "peakRssMb": 24299, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 227786, "output": 3492 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-author-linear", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 1 }, "trialId": "craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-29T08-00-04-504Z-oagz", "success": true, "includedInCapability": true, "durationMs": 2184123, "timing": { "startedAt": "2026-10-29T08:00:04.504Z", "finishedAt": "2026-10-29T08:36:28.627Z", "timeToFirstArtifactMs": 1408138, "timeToLastArtifactWriteMs": 2168979, "timeToFirstTokenMs": 41503, "firstTurnTtftMs": 39831, "timeToFirstToolCallMs": 55108 }, "metrics": { "process": { "sampleCount": 433, "peakRssMb": 19720, "peakCpuPercent": 109.6, "firstAtMs": 0, "lastAtMs": 2167931 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 433, "peakUsedMb": 91018, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 108261, "totalOutputTokens": 40830, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 233794, "todayTokensCached": 228812, "todayTokensOut": 2789, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 233794, "totalTokensCached": 228812, "totalTokensOut": 2789, "totalCost": 0, "medianOutputTokensPerSec": 23.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.4, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 15813, "cachedInputTokens": 15727, "outputTokens": 80, "cost": 0, "durationMs": 336608, "outputTokensPerSec": 28.295990387026695, "at": "2026-10-29T08:36:13.483Z" }, "lastUpdated": "2026-10-29T08:36:13.483Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T08:36:18.527Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2172957, "meanTokensPerSec": 21, "genTokensPerSec": 21, "genTokens": 40830, "promptTokensPerSec": null, "peakRssMb": 19720, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 233794, "output": 2789 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 495, "sampleCount": 11, "structuredSampleCount": 0, "firstTokenSampleCount": 11, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-route-multi", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 1 }, "trialId": "craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-29T08-36-51-158Z-69bz", "success": false, "includedInCapability": true, "durationMs": 1206128, "timing": { "startedAt": "2026-10-29T08:36:51.158Z", "finishedAt": "2026-10-29T08:56:57.286Z", "timeToFirstArtifactMs": 1050678, "timeToLastArtifactWriteMs": 1203384, "timeToFirstTokenMs": 17197, "firstTurnTtftMs": 15470, "timeToFirstToolCallMs": 50855 }, "metrics": { "process": { "sampleCount": 240, "peakRssMb": 11002, "peakCpuPercent": 123.6, "firstAtMs": 0, "lastAtMs": 1200143 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 240, "peakUsedMb": 76298, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 77020, "totalOutputTokens": 36558, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 149847, "todayTokensCached": 110176, "todayTokensOut": 30076, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 149847, "totalTokensCached": 110176, "totalTokensOut": 30076, "totalCost": 0, "medianOutputTokensPerSec": 35.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.3, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13595, "cachedInputTokens": 12766, "outputTokens": 277, "cost": 0, "durationMs": 8753, "outputTokensPerSec": 35.490877953247235, "at": "2026-10-29T08:56:54.542Z" }, "lastUpdated": "2026-10-29T08:56:54.542Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T08:56:56.171Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1203951, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 36558, "promptTokensPerSec": null, "peakRssMb": 11002, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 149847, "output": 30076 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 880, "sampleCount": 15, "structuredSampleCount": 0, "firstTokenSampleCount": 15, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-route-multi", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 1 }, "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-29T08-57-18-583Z-761o", "success": false, "includedInCapability": true, "durationMs": 416305, "timing": { "startedAt": "2026-10-29T08:57:18.583Z", "finishedAt": "2026-10-29T09:04:14.888Z", "timeToFirstArtifactMs": 160006, "timeToLastArtifactWriteMs": 413105, "timeToFirstTokenMs": 15808, "firstTurnTtftMs": 13873, "timeToFirstToolCallMs": 25387 }, "metrics": { "process": { "sampleCount": 83, "peakRssMb": 25535, "peakCpuPercent": 125.9, "firstAtMs": 0, "lastAtMs": 411537 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 83, "peakUsedMb": 88701, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 102352, "totalOutputTokens": 14562, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 89615, "todayTokensCached": 67965, "todayTokensOut": 2890, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 89615, "totalTokensCached": 67965, "totalTokensOut": 2890, "totalCost": 0, "medianOutputTokensPerSec": 56.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 56.7, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 20714, "cachedInputTokens": 19678, "outputTokens": 2349, "cost": 0, "durationMs": 51409, "outputTokensPerSec": 46.761568220487725, "at": "2026-10-29T09:02:42.995Z" }, "lastUpdated": "2026-10-29T09:02:42.995Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T09:04:12.955Z" } }, "source": "engine-log" }, "derived": { "durationMs": 413323, "meanTokensPerSec": 55.46, "genTokensPerSec": 55.46, "genTokens": 14562, "promptTokensPerSec": 906.87, "peakRssMb": 25535, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 89615, "output": 2890 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-route-multi", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 1 }, "trialId": "craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-29T09-04-39-277Z-5er1", "success": true, "includedInCapability": true, "durationMs": 1431991, "timing": { "startedAt": "2026-10-29T09:04:39.278Z", "finishedAt": "2026-10-29T09:28:31.269Z", "timeToFirstArtifactMs": 369977, "timeToLastArtifactWriteMs": 1418199, "timeToFirstTokenMs": 30533, "firstTurnTtftMs": 28815, "timeToFirstToolCallMs": 50232 }, "metrics": { "process": { "sampleCount": 284, "peakRssMb": 20535, "peakCpuPercent": 137.9, "firstAtMs": 0, "lastAtMs": 1420271 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 284, "peakUsedMb": 92584, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 155287, "totalOutputTokens": 23320, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 280522, "todayTokensCached": 262803, "todayTokensOut": 7838, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 280522, "totalTokensCached": 262803, "totalTokensOut": 7838, "totalCost": 0, "medianOutputTokensPerSec": 26.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.4, "turns": 18 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7830, "cachedInputTokens": 7745, "outputTokens": 69, "cost": 0, "durationMs": 109245, "outputTokensPerSec": 27.760952620751777, "at": "2026-10-29T09:28:20.543Z" }, "lastUpdated": "2026-10-29T09:28:20.543Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T09:28:24.162Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1423819, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 23320, "promptTokensPerSec": null, "peakRssMb": 20535, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 280522, "output": 7838 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 472, "sampleCount": 19, "structuredSampleCount": 0, "firstTokenSampleCount": 19, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-route-multi", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 1 }, "trialId": "craftbook-route-multi-qwen3-8-27b-q4-2026-10-29T09-28-57-101Z-a4oz", "success": true, "includedInCapability": true, "durationMs": 2084956, "timing": { "startedAt": "2026-10-29T09:28:57.101Z", "finishedAt": "2026-10-29T10:03:42.057Z", "timeToFirstArtifactMs": 314363, "timeToLastArtifactWriteMs": 1914837, "timeToFirstTokenMs": 35000, "firstTurnTtftMs": 33086, "timeToFirstToolCallMs": 78625 }, "metrics": { "process": { "sampleCount": 415, "peakRssMb": 51264, "peakCpuPercent": 150.6, "firstAtMs": 0, "lastAtMs": 2077386 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 415, "peakUsedMb": 113647, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 350560, "totalOutputTokens": 25129, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 302748, "todayTokensCached": 180578, "todayTokensOut": 6033, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 302748, "totalTokensCached": 180578, "totalTokensOut": 6033, "totalCost": 0, "medianOutputTokensPerSec": 24, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24, "turns": 16 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8917, "cachedInputTokens": 8492, "outputTokens": 182, "cost": 0, "durationMs": 364354, "outputTokensPerSec": 23.702333265543786, "at": "2026-10-29T10:03:38.640Z" }, "lastUpdated": "2026-10-29T10:03:38.640Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T10:03:39.837Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2081688, "meanTokensPerSec": 23.82, "genTokensPerSec": 23.82, "genTokens": 25129, "promptTokensPerSec": 365.53, "peakRssMb": 51264, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 302748, "output": 6033 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-export-generalize", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 1 }, "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-29T10-04-04-005Z-uul6", "success": false, "includedInCapability": true, "durationMs": 989254, "timing": { "startedAt": "2026-10-29T10:04:04.005Z", "finishedAt": "2026-10-29T10:20:33.259Z", "timeToFirstArtifactMs": 83474, "timeToLastArtifactWriteMs": 274736, "timeToFirstTokenMs": 20052, "firstTurnTtftMs": 18192, "timeToFirstToolCallMs": 34639 }, "metrics": { "process": { "sampleCount": 197, "peakRssMb": 26277, "peakCpuPercent": 120, "firstAtMs": 1, "lastAtMs": 985530 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 197, "peakUsedMb": 88287, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 70824, "totalOutputTokens": 14411, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 231418, "todayTokensCached": 198467, "todayTokensOut": 9079, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 231418, "totalTokensCached": 198467, "totalTokensOut": 9079, "totalCost": 0, "medianOutputTokensPerSec": 51.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 51.2, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 21428, "cachedInputTokens": 21068, "outputTokens": 2018, "cost": 0, "durationMs": 31045, "outputTokensPerSec": 66.33498952650433, "at": "2026-10-29T10:15:26.029Z" }, "lastUpdated": "2026-10-29T10:15:26.029Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T10:20:33.128Z" } }, "source": "engine-log" }, "derived": { "durationMs": 988076, "meanTokensPerSec": 56.16, "genTokensPerSec": 56.16, "genTokens": 14411, "promptTokensPerSec": 894.13, "peakRssMb": 26277, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 231418, "output": 9079 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-export-generalize", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 1 }, "trialId": "craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-29T10-20-54-840Z-oj3a", "success": false, "includedInCapability": true, "durationMs": 7657533, "timing": { "startedAt": "2026-10-29T10:20:54.841Z", "finishedAt": "2026-10-29T12:28:32.374Z", "timeToFirstArtifactMs": 946176, "timeToLastArtifactWriteMs": 6858554, "timeToFirstTokenMs": 31650, "firstTurnTtftMs": 30111, "timeToFirstToolCallMs": 150876 }, "metrics": { "process": { "sampleCount": 1524, "peakRssMb": 24651, "peakCpuPercent": 171.8, "firstAtMs": 0, "lastAtMs": 7652625 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1524, "peakUsedMb": 96897, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 139624, "totalOutputTokens": 121882, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 276353, "todayTokensCached": 272047, "todayTokensOut": 32551, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 276353, "totalTokensCached": 272047, "totalTokensOut": 32551, "totalCost": 0, "medianOutputTokensPerSec": 25.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.1, "turns": 15 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 6550, "cachedInputTokens": 6466, "outputTokens": 64, "cost": 0, "durationMs": 12612, "outputTokensPerSec": 49.044981981676955, "at": "2026-10-29T12:16:31.566Z" }, "lastUpdated": "2026-10-29T12:16:31.566Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T12:28:30.464Z" } }, "source": "engine-log" }, "derived": { "durationMs": 7654661, "meanTokensPerSec": 19, "genTokensPerSec": 19, "genTokens": 121882, "promptTokensPerSec": null, "peakRssMb": 24651, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 276353, "output": 32551 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 532, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-export-generalize", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 1 }, "trialId": "craftbook-export-generalize-qwen3-8-27b-q4-2026-10-29T12-28-54-403Z-godl", "success": true, "includedInCapability": true, "durationMs": 1604266, "timing": { "startedAt": "2026-10-29T12:28:54.403Z", "finishedAt": "2026-10-29T12:55:38.669Z", "timeToFirstArtifactMs": 879374, "timeToLastArtifactWriteMs": 1596146, "timeToFirstTokenMs": 50840, "firstTurnTtftMs": 48978, "timeToFirstToolCallMs": 103737 }, "metrics": { "process": { "sampleCount": 319, "peakRssMb": 48264, "peakCpuPercent": 88.2, "firstAtMs": 0, "lastAtMs": 1595686 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 319, "peakUsedMb": 113784, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 115301, "totalOutputTokens": 25674, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 182198, "todayTokensCached": 177094, "todayTokensOut": 2647, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 182198, "totalTokensCached": 177094, "totalTokensOut": 2647, "totalCost": 0, "medianOutputTokensPerSec": 20, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 20, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9895, "cachedInputTokens": 9568, "outputTokens": 77, "cost": 0, "durationMs": 259094, "outputTokensPerSec": 25.19206463223066, "at": "2026-10-29T12:55:30.549Z" }, "lastUpdated": "2026-10-29T12:55:30.549Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T12:55:34.923Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1599469, "meanTokensPerSec": 21.4, "genTokensPerSec": 21.4, "genTokens": 25674, "promptTokensPerSec": 321.15, "peakRssMb": 48264, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 182198, "output": 2647 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-export-generalize", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 1 }, "trialId": "craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-29T12-56-00-732Z-h88c", "success": false, "includedInCapability": true, "durationMs": 1390718, "timing": { "startedAt": "2026-10-29T12:56:00.732Z", "finishedAt": "2026-10-29T13:19:11.450Z", "timeToFirstArtifactMs": 118057, "timeToLastArtifactWriteMs": 1384483, "timeToFirstTokenMs": 24421, "firstTurnTtftMs": 22762, "timeToFirstToolCallMs": 55013 }, "metrics": { "process": { "sampleCount": 277, "peakRssMb": 10975, "peakCpuPercent": 138.2, "firstAtMs": 0, "lastAtMs": 1388000 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 277, "peakUsedMb": 76591, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 35914, "totalOutputTokens": 23236, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 476080, "todayTokensCached": 468026, "todayTokensOut": 4964, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 476080, "totalTokensCached": 468026, "totalTokensOut": 4964, "totalCost": 0, "medianOutputTokensPerSec": 33.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 33.4, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 35130, "cachedInputTokens": 34950, "outputTokens": 163, "cost": 0, "durationMs": 72850, "outputTokensPerSec": 32.884186765426485, "at": "2026-10-29T13:19:05.215Z" }, "lastUpdated": "2026-10-29T13:19:05.215Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T13:19:10.330Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1388546, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 23236, "promptTokensPerSec": null, "peakRssMb": 10975, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 476080, "output": 4964 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 490, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-author-params", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 1 }, "trialId": "craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-29T13-19-36-196Z-24u6", "success": true, "includedInCapability": true, "durationMs": 1931122, "timing": { "startedAt": "2026-10-29T13:19:36.196Z", "finishedAt": "2026-10-29T13:51:47.318Z", "timeToFirstArtifactMs": 1040776, "timeToLastArtifactWriteMs": 1924216, "timeToFirstTokenMs": 43102, "firstTurnTtftMs": 41313, "timeToFirstToolCallMs": 51634 }, "metrics": { "process": { "sampleCount": 384, "peakRssMb": 20101, "peakCpuPercent": 187.8, "firstAtMs": 0, "lastAtMs": 1923626 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 384, "peakUsedMb": 92217, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 83952, "totalOutputTokens": 29628, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 139651, "todayTokensCached": 138094, "todayTokensOut": 5163, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 139651, "totalTokensCached": 138094, "totalTokensOut": 5163, "totalCost": 0, "medianOutputTokensPerSec": 32, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 3781, "cachedInputTokens": 3527, "outputTokens": 112, "cost": 0, "durationMs": 46556, "outputTokensPerSec": 39.95770782682488, "at": "2026-10-29T13:51:14.096Z" }, "lastUpdated": "2026-10-29T13:51:14.096Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T13:51:41.764Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1924402, "meanTokensPerSec": 18, "genTokensPerSec": 18, "genTokens": 29628, "promptTokensPerSec": null, "peakRssMb": 20101, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 139651, "output": 5163 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 509, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-author-params", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 1 }, "trialId": "craftbook-author-params-qwen3-8-27b-q4-2026-10-29T13-52-10-021Z-c189", "success": true, "includedInCapability": true, "durationMs": 1202939, "timing": { "startedAt": "2026-10-29T13:52:10.022Z", "finishedAt": "2026-10-29T14:12:12.961Z", "timeToFirstArtifactMs": 1044755, "timeToLastArtifactWriteMs": 1196562, "timeToFirstTokenMs": 49491, "firstTurnTtftMs": 47515, "timeToFirstToolCallMs": 58303 }, "metrics": { "process": { "sampleCount": 239, "peakRssMb": 53295, "peakCpuPercent": 108.5, "firstAtMs": 0, "lastAtMs": 1194285 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 239, "peakUsedMb": 118473, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 103072, "totalOutputTokens": 20099, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 170578, "todayTokensCached": 165386, "todayTokensOut": 2293, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 170578, "totalTokensCached": 165386, "totalTokensOut": 2293, "totalCost": 0, "medianOutputTokensPerSec": 27.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27.3, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 38731, "cachedInputTokens": 38049, "outputTokens": 290, "cost": 0, "durationMs": 207675, "outputTokensPerSec": 20.247324218020665, "at": "2026-10-29T14:11:22.156Z" }, "lastUpdated": "2026-10-29T14:11:22.156Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T14:12:08.495Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1197328, "meanTokensPerSec": 23.21, "genTokensPerSec": 23.21, "genTokens": 20099, "promptTokensPerSec": 354.57, "peakRssMb": 53295, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 170578, "output": 2293 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-author-params", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 1 }, "trialId": "craftbook-author-params-mlx-gemma4-12b-q4-2026-10-29T14-12-35-123Z-ovs1", "success": false, "includedInCapability": true, "durationMs": 839550, "timing": { "startedAt": "2026-10-29T14:12:35.124Z", "finishedAt": "2026-10-29T14:26:34.674Z", "timeToFirstArtifactMs": 564236, "timeToLastArtifactWriteMs": 804485, "timeToFirstTokenMs": 21297, "firstTurnTtftMs": 19622, "timeToFirstToolCallMs": 41289 }, "metrics": { "process": { "sampleCount": 167, "peakRssMb": 10778, "peakCpuPercent": 119.6, "firstAtMs": 0, "lastAtMs": 833859 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 167, "peakUsedMb": 77155, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 32057, "totalOutputTokens": 21278, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 461311, "todayTokensCached": 455320, "todayTokensOut": 3447, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 461311, "totalTokensCached": 455320, "totalTokensOut": 3447, "totalCost": 0, "medianOutputTokensPerSec": 37.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.4, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 31257, "cachedInputTokens": 31006, "outputTokens": 854, "cost": 0, "durationMs": 31160, "outputTokensPerSec": 28.04538191619223, "at": "2026-10-29T14:26:30.807Z" }, "lastUpdated": "2026-10-29T14:26:30.807Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T14:26:33.613Z" } }, "source": "engine-log" }, "derived": { "durationMs": 837426, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 21278, "promptTokensPerSec": null, "peakRssMb": 10778, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 461311, "output": 3447 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 461.5, "sampleCount": 18, "structuredSampleCount": 0, "firstTokenSampleCount": 18, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-author-params", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 1 }, "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-29T14-26-56-547Z-v2ns", "success": false, "includedInCapability": false, "durationMs": 1126851, "timing": { "startedAt": "2026-10-29T14:26:56.547Z", "finishedAt": "2026-10-29T14:45:43.398Z", "timeToFirstArtifactMs": 332522, "timeToLastArtifactWriteMs": 554724, "timeToFirstTokenMs": 22287, "firstTurnTtftMs": 20426, "timeToFirstToolCallMs": 35761 }, "metrics": { "process": { "sampleCount": 224, "peakRssMb": 26952, "peakCpuPercent": 124.1, "firstAtMs": 0, "lastAtMs": 1120791 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 224, "peakUsedMb": 90969, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 109333, "totalOutputTokens": 19999, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 130548, "todayTokensCached": 127401, "todayTokensOut": 3537, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 130548, "totalTokensCached": 127401, "totalTokensOut": 3537, "totalCost": 0, "medianOutputTokensPerSec": 42.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.5, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9654, "cachedInputTokens": 9362, "outputTokens": 856, "cost": 0, "durationMs": 9295, "outputTokensPerSec": 96.46134652819255, "at": "2026-10-29T14:40:53.437Z" }, "lastUpdated": "2026-10-29T14:40:53.437Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T14:45:42.457Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1124861, "meanTokensPerSec": 59.35, "genTokensPerSec": 59.35, "genTokens": 19999, "promptTokensPerSec": 848.44, "peakRssMb": 26952, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 130548, "output": 3537 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-author-gate-script", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 1 }, "trialId": "craftbook-author-gate-script-qwen3-8-27b-q4-2026-10-29T16-44-56-719Z-bi1u", "success": false, "includedInCapability": true, "durationMs": 5432330, "timing": { "startedAt": "2026-10-29T16:44:56.719Z", "finishedAt": "2026-10-29T18:15:29.049Z", "timeToFirstArtifactMs": 2174616, "timeToLastArtifactWriteMs": 4815144, "timeToFirstTokenMs": 39039, "firstTurnTtftMs": 37209, "timeToFirstToolCallMs": 43001 }, "metrics": { "process": { "sampleCount": 1081, "peakRssMb": 48733, "peakCpuPercent": 167.9, "firstAtMs": 0, "lastAtMs": 5419286 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1081, "peakUsedMb": 115107, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 846385, "totalOutputTokens": 27531, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 137695, "todayTokensCached": 16368, "todayTokensOut": 1412, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 137695, "totalTokensCached": 16368, "totalTokensOut": 1412, "totalCost": 0, "medianOutputTokensPerSec": 33.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.8, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 10506, "cachedInputTokens": 10379, "outputTokens": 298, "cost": 0, "durationMs": 3223034, "outputTokensPerSec": 30.907233447745597, "at": "2026-10-29T18:05:11.862Z" }, "lastUpdated": "2026-10-29T18:05:11.862Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T18:15:21.844Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5424072, "meanTokensPerSec": 19.49, "genTokensPerSec": 19.49, "genTokens": 27531, "promptTokensPerSec": 253.91, "peakRssMb": 48733, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 137695, "output": 1412 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-author-gate-script", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 1 }, "trialId": "craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-29T18-15-52-291Z-hb1t", "success": false, "includedInCapability": true, "durationMs": 396988, "timing": { "startedAt": "2026-10-29T18:15:52.291Z", "finishedAt": "2026-10-29T18:22:29.279Z", "timeToFirstArtifactMs": 232817, "timeToLastArtifactWriteMs": 389615, "timeToFirstTokenMs": 21390, "firstTurnTtftMs": 19479, "timeToFirstToolCallMs": 36880 }, "metrics": { "process": { "sampleCount": 79, "peakRssMb": 10810, "peakCpuPercent": 112.7, "firstAtMs": 0, "lastAtMs": 391737 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 76755, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 71485, "totalOutputTokens": 7904, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 244295, "todayTokensCached": 208287, "todayTokensOut": 3191, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 244295, "totalTokensCached": 208287, "totalTokensOut": 3191, "totalCost": 0, "medianOutputTokensPerSec": 36.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.6, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7459, "cachedInputTokens": 0, "outputTokens": 459, "cost": 0, "durationMs": 21216, "outputTokensPerSec": 34.75113726327703, "at": "2026-10-29T18:22:21.906Z" }, "lastUpdated": "2026-10-29T18:22:21.906Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T18:22:27.086Z" } }, "source": "engine-log" }, "derived": { "durationMs": 393525, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 7904, "promptTokensPerSec": null, "peakRssMb": 10810, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 244295, "output": 3191 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 585.5, "sampleCount": 16, "structuredSampleCount": 0, "firstTokenSampleCount": 16, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-author-gate-script", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 1 }, "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-29T18-22-51-537Z-6n0y", "success": false, "includedInCapability": true, "durationMs": 1337701, "timing": { "startedAt": "2026-10-29T18:22:51.537Z", "finishedAt": "2026-10-29T18:45:09.238Z", "timeToFirstArtifactMs": 892056, "timeToLastArtifactWriteMs": 1332064, "timeToFirstTokenMs": 20034, "firstTurnTtftMs": 18159, "timeToFirstToolCallMs": 26416 }, "metrics": { "process": { "sampleCount": 266, "peakRssMb": 29106, "peakCpuPercent": 110.6, "firstAtMs": 0, "lastAtMs": 1330232 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 266, "peakUsedMb": 97705, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 234090, "totalOutputTokens": 51149, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 327726, "todayTokensCached": 266377, "todayTokensOut": 7356, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 327726, "totalTokensCached": 266377, "totalTokensOut": 7356, "totalCost": 0, "medianOutputTokensPerSec": 70.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.7, "turns": 22 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10863, "cachedInputTokens": 10734, "outputTokens": 31, "cost": 0, "durationMs": 98946, "outputTokensPerSec": 118.04284955438824, "at": "2026-10-29T18:45:03.601Z" }, "lastUpdated": "2026-10-29T18:45:03.601Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T18:45:06.854Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1334280, "meanTokensPerSec": 63.6, "genTokensPerSec": 63.6, "genTokens": 51149, "promptTokensPerSec": 925.82, "peakRssMb": 29106, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 327726, "output": 7356 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-author-gate-script", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 1 }, "trialId": "craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-29T18-45-30-573Z-tp7x", "success": false, "includedInCapability": true, "durationMs": 3476092, "timing": { "startedAt": "2026-10-29T18:45:30.573Z", "finishedAt": "2026-10-29T19:43:26.665Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 39493, "firstTurnTtftMs": 37810, "timeToFirstToolCallMs": 54063 }, "metrics": { "process": { "sampleCount": 691, "peakRssMb": 18399, "peakCpuPercent": 128.7, "firstAtMs": 0, "lastAtMs": 3462703 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 691, "peakUsedMb": 96701, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 152454, "totalOutputTokens": 38107, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 152070, "todayTokensCached": 151875, "todayTokensOut": 4098, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 152070, "totalTokensCached": 151875, "totalTokensOut": 4098, "totalCost": 0, "medianOutputTokensPerSec": 10.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 10.7, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 152070, "cachedInputTokens": 151875, "outputTokens": 4098, "cost": 0, "durationMs": 3248785, "outputTokensPerSec": 10.659262764072764, "at": "2026-10-29T19:39:41.041Z" }, "lastUpdated": "2026-10-29T19:39:41.041Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T19:43:19.378Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3467730, "meanTokensPerSec": 12, "genTokensPerSec": 12, "genTokens": 38107, "promptTokensPerSec": null, "peakRssMb": 18399, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 152070, "output": 4098 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 330, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-mlx-complex-work-craftbook-author-fanout", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 1 }, "trialId": "craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-29T19-43-48-872Z-euv1", "success": false, "includedInCapability": true, "durationMs": 839994, "timing": { "startedAt": "2026-10-29T19:43:48.873Z", "finishedAt": "2026-10-29T19:57:48.867Z", "timeToFirstArtifactMs": 131138, "timeToLastArtifactWriteMs": 734716, "timeToFirstTokenMs": 20826, "firstTurnTtftMs": 19157, "timeToFirstToolCallMs": 33492 }, "metrics": { "process": { "sampleCount": 167, "peakRssMb": 11140, "peakCpuPercent": 115.2, "firstAtMs": 0, "lastAtMs": 834361 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 167, "peakUsedMb": 80354, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 56682, "totalOutputTokens": 13308, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 215919, "todayTokensCached": 211254, "todayTokensOut": 2539, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 215919, "totalTokensCached": 211254, "totalTokensOut": 2539, "totalCost": 0, "medianOutputTokensPerSec": 42.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.4, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9919, "cachedInputTokens": 9802, "outputTokens": 163, "cost": 0, "durationMs": 162153, "outputTokensPerSec": 42.419539856228546, "at": "2026-10-29T19:57:21.089Z" }, "lastUpdated": "2026-10-29T19:57:21.089Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T19:57:47.349Z" } }, "source": "engine-log" }, "derived": { "durationMs": 837416, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 13308, "promptTokensPerSec": null, "peakRssMb": 11140, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 215919, "output": 2539 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 652, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-gemma-llamacpp-complex-work-craftbook-author-fanout", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 1 }, "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-29T19-58-10-845Z-mu9j", "success": false, "includedInCapability": true, "durationMs": 1386756, "timing": { "startedAt": "2026-10-29T19:58:10.845Z", "finishedAt": "2026-10-29T20:21:17.601Z", "timeToFirstArtifactMs": 293091, "timeToLastArtifactWriteMs": 888622, "timeToFirstTokenMs": 18773, "firstTurnTtftMs": 16874, "timeToFirstToolCallMs": 25951 }, "metrics": { "process": { "sampleCount": 275, "peakRssMb": 28838, "peakCpuPercent": 101, "firstAtMs": 0, "lastAtMs": 1376966 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 275, "peakUsedMb": 91839, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 122880, "totalOutputTokens": 28597, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 23, "todayTokensIn": 415241, "todayTokensCached": 409734, "todayTokensOut": 8118, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 415241, "totalTokensCached": 409734, "totalTokensOut": 8118, "totalCost": 0, "medianOutputTokensPerSec": 48.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 48.4, "turns": 23 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8158, "cachedInputTokens": 7973, "outputTokens": 302, "cost": 0, "durationMs": 3734, "outputTokensPerSec": 88.94520443508566, "at": "2026-10-29T20:17:46.702Z" }, "lastUpdated": "2026-10-29T20:17:46.702Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T20:21:13.686Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1381795, "meanTokensPerSec": 50.82, "genTokensPerSec": 50.82, "genTokens": 28597, "promptTokensPerSec": 947.91, "peakRssMb": 28838, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 415241, "output": 8118 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r1-qwen-mlx-complex-work-craftbook-author-fanout", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 1 }, "trialId": "craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-29T20-21-38-892Z-yv8n", "success": false, "includedInCapability": true, "durationMs": 4951692, "timing": { "startedAt": "2026-10-29T20:21:38.893Z", "finishedAt": "2026-10-29T21:44:10.585Z", "timeToFirstArtifactMs": 3119334, "timeToLastArtifactWriteMs": 4846459, "timeToFirstTokenMs": 31499, "firstTurnTtftMs": 29938, "timeToFirstToolCallMs": 44859 }, "metrics": { "process": { "sampleCount": 985, "peakRssMb": 19786, "peakCpuPercent": 141.2, "firstAtMs": 0, "lastAtMs": 4938982 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 985, "peakUsedMb": 96620, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 170211, "totalOutputTokens": 68850, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 440342, "todayTokensCached": 422724, "todayTokensOut": 5228, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 440342, "totalTokensCached": 422724, "totalTokensOut": 5228, "totalCost": 0, "medianOutputTokensPerSec": 15.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 15.8, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 116030, "cachedInputTokens": 115475, "outputTokens": 857, "cost": 0, "durationMs": 155663, "outputTokensPerSec": 11.725416745277494, "at": "2026-10-29T21:41:55.086Z" }, "lastUpdated": "2026-10-29T21:41:55.086Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T21:44:00.625Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4940769, "meanTokensPerSec": 15, "genTokensPerSec": 15, "genTokens": 68850, "promptTokensPerSec": null, "peakRssMb": 19786, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 440342, "output": 5228 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 259, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r1-qwen-llamacpp-complex-work-craftbook-author-fanout", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 1 }, "trialId": "craftbook-author-fanout-qwen3-8-27b-q4-2026-10-29T21-44-32-345Z-x8yr", "success": false, "includedInCapability": true, "durationMs": 1780520, "timing": { "startedAt": "2026-10-29T21:44:32.345Z", "finishedAt": "2026-10-29T22:14:12.865Z", "timeToFirstArtifactMs": 692850, "timeToLastArtifactWriteMs": 954265, "timeToFirstTokenMs": 52339, "firstTurnTtftMs": 50482, "timeToFirstToolCallMs": 68552 }, "metrics": { "process": { "sampleCount": 354, "peakRssMb": 51291, "peakCpuPercent": 95.1, "firstAtMs": 0, "lastAtMs": 1771352 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 354, "peakUsedMb": 118351, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 163635, "totalOutputTokens": 23661, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 144089, "todayTokensCached": 135015, "todayTokensOut": 2952, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 144089, "totalTokensCached": 135015, "totalTokensOut": 2952, "totalCost": 0, "medianOutputTokensPerSec": 25.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.3, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 6636, "cachedInputTokens": 6398, "outputTokens": 57, "cost": 0, "durationMs": 326994, "outputTokensPerSec": 47.568485878105754, "at": "2026-10-29T22:06:56.246Z" }, "lastUpdated": "2026-10-29T22:06:56.246Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:14:09.301Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1775904, "meanTokensPerSec": 20.3, "genTokensPerSec": 20.3, "genTokens": 23661, "promptTokensPerSec": 309.52, "peakRssMb": 51291, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 144089, "output": 2952 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-core-tictactoe", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "tictactoe", "repetition": 2 }, "trialId": "tictactoe-qwen3-8-27b-q4-2026-10-29T22-14-39-215Z-3ofq", "success": true, "includedInCapability": true, "durationMs": 111825, "timing": { "startedAt": "2026-10-29T22:14:39.216Z", "finishedAt": "2026-10-29T22:16:31.041Z", "timeToFirstArtifactMs": 105695, "timeToLastArtifactWriteMs": 105700, "timeToFirstTokenMs": 25393, "firstTurnTtftMs": 23509, "timeToFirstToolCallMs": 35965 }, "metrics": { "process": { "sampleCount": 22, "peakRssMb": 39610, "peakCpuPercent": 103.1, "firstAtMs": 0, "lastAtMs": 105378 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 22, "peakUsedMb": 102520, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13252, "totalOutputTokens": 2143, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22280, "todayTokensCached": 21250, "todayTokensOut": 542, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22280, "totalTokensCached": 21250, "totalTokensOut": 542, "totalCost": 0, "medianOutputTokensPerSec": 30.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.7, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7604, "cachedInputTokens": 7406, "outputTokens": 207, "cost": 0, "durationMs": 9447, "outputTokensPerSec": 24.789234348396327, "at": "2026-10-29T22:15:38.450Z" }, "lastUpdated": "2026-10-29T22:15:38.450Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:16:27.425Z" } }, "source": "engine-log" }, "derived": { "durationMs": 107159, "meanTokensPerSec": 35.17, "genTokensPerSec": 35.17, "genTokens": 2143, "promptTokensPerSec": 408.48, "peakRssMb": 39610, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22280, "output": 542 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-tictactoe", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "tictactoe", "repetition": 2 }, "trialId": "tictactoe-mlx-gemma4-12b-q4-2026-10-29T22-17-16-394Z-jaso", "success": true, "includedInCapability": true, "durationMs": 90890, "timing": { "startedAt": "2026-10-29T22:17:16.395Z", "finishedAt": "2026-10-29T22:18:47.285Z", "timeToFirstArtifactMs": 38453, "timeToLastArtifactWriteMs": 87549, "timeToFirstTokenMs": 16559, "firstTurnTtftMs": 14871, "timeToFirstToolCallMs": 27040 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 10738, "peakCpuPercent": 108, "firstAtMs": 0, "lastAtMs": 85320 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 79689, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 28368, "totalOutputTokens": 1887, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 28320, "todayTokensCached": 20074, "todayTokensOut": 214, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 28320, "totalTokensCached": 20074, "totalTokensOut": 214, "totalCost": 0, "medianOutputTokensPerSec": 47.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 47.6, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11285, "cachedInputTokens": 9786, "outputTokens": 31, "cost": 0, "durationMs": 48633, "outputTokensPerSec": 44.73934581571396, "at": "2026-10-29T22:18:43.944Z" }, "lastUpdated": "2026-10-29T22:18:43.944Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:18:44.564Z" } }, "source": "engine-log" }, "derived": { "durationMs": 87103, "meanTokensPerSec": 42, "genTokensPerSec": 42, "genTokens": 1887, "promptTokensPerSec": null, "peakRssMb": 10738, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 28320, "output": 214 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1393, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-tictactoe", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "tictactoe", "repetition": 2 }, "trialId": "tictactoe-gemma4-12b-q4-2026-10-29T22-19-34-173Z-9cqy", "success": true, "includedInCapability": true, "durationMs": 45508, "timing": { "startedAt": "2026-10-29T22:19:34.173Z", "finishedAt": "2026-10-29T22:20:19.681Z", "timeToFirstArtifactMs": 40591, "timeToLastArtifactWriteMs": 40595, "timeToFirstTokenMs": 11080, "firstTurnTtftMs": 9170, "timeToFirstToolCallMs": 17032 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 18061, "peakCpuPercent": 106.8, "firstAtMs": 0, "lastAtMs": 40151 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 82074, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13567, "totalOutputTokens": 1984, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6949, "todayTokensCached": 6777, "todayTokensOut": 38, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6949, "totalTokensCached": 6777, "totalTokensOut": 38, "totalCost": 0, "medianOutputTokensPerSec": 69.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 69.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6949, "cachedInputTokens": 6777, "outputTokens": 38, "cost": 0, "durationMs": 16791, "outputTokensPerSec": 69.39794469546493, "at": "2026-10-29T22:19:52.874Z" }, "lastUpdated": "2026-10-29T22:19:52.874Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:20:17.159Z" } }, "source": "engine-log" }, "derived": { "durationMs": 41933, "meanTokensPerSec": 106.42, "genTokensPerSec": 106.42, "genTokens": 1984, "promptTokensPerSec": 1293.47, "peakRssMb": 18061, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6949, "output": 38 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-tictactoe", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "tictactoe", "repetition": 2 }, "trialId": "tictactoe-mlx-qwen3-8-27b-q4-2026-10-29T22-21-06-782Z-1qtd", "success": true, "includedInCapability": true, "durationMs": 84069, "timing": { "startedAt": "2026-10-29T22:21:06.782Z", "finishedAt": "2026-10-29T22:22:30.851Z", "timeToFirstArtifactMs": 75241, "timeToLastArtifactWriteMs": 75245, "timeToFirstTokenMs": 23306, "firstTurnTtftMs": 21620, "timeToFirstToolCallMs": 29599 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 18397, "peakCpuPercent": 123.8, "firstAtMs": 0, "lastAtMs": 75271 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 96654, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17140, "totalOutputTokens": 1691, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 17124, "todayTokensCached": 11611, "todayTokensOut": 1654, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 17124, "totalTokensCached": 11611, "totalTokensOut": 1654, "totalCost": 0, "medianOutputTokensPerSec": 41.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 41.7, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5341, "cachedInputTokens": 0, "outputTokens": 1551, "cost": 0, "durationMs": 38756, "outputTokensPerSec": 48.223244500912266, "at": "2026-10-29T22:22:22.027Z" }, "lastUpdated": "2026-10-29T22:22:22.027Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:22:24.904Z" } }, "source": "engine-log" }, "derived": { "durationMs": 77052, "meanTokensPerSec": 49, "genTokensPerSec": 49, "genTokens": 1691, "promptTokensPerSec": null, "peakRssMb": 18397, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 17124, "output": 1654 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 805.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-core-petshop", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "petshop", "repetition": 2 }, "trialId": "petshop-mlx-gemma4-12b-q4-2026-10-29T22-23-14-365Z-yrrm", "success": false, "includedInCapability": true, "durationMs": 830449, "timing": { "startedAt": "2026-10-29T22:23:14.366Z", "finishedAt": "2026-10-29T22:37:04.815Z", "timeToFirstArtifactMs": 109636, "timeToLastArtifactWriteMs": 823637, "timeToFirstTokenMs": 16463, "firstTurnTtftMs": 14784, "timeToFirstToolCallMs": 38707 }, "metrics": { "process": { "sampleCount": 165, "peakRssMb": 12274, "peakCpuPercent": 139.9, "firstAtMs": 0, "lastAtMs": 823631 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 165, "peakUsedMb": 78920, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 132852, "totalOutputTokens": 18218, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 173342, "todayTokensCached": 132902, "todayTokensOut": 12152, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 173342, "totalTokensCached": 132902, "totalTokensOut": 12152, "totalCost": 0, "medianOutputTokensPerSec": 34.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.4, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12532, "cachedInputTokens": 1485, "outputTokens": 1323, "cost": 0, "durationMs": 68355, "outputTokensPerSec": 33.20552030312575, "at": "2026-10-29T22:36:58.003Z" }, "lastUpdated": "2026-10-29T22:36:58.003Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:36:59.596Z" } }, "source": "engine-log" }, "derived": { "durationMs": 824164, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 18218, "promptTokensPerSec": null, "peakRssMb": 12274, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 173342, "output": 12152 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 637, "sampleCount": 19, "structuredSampleCount": 0, "firstTokenSampleCount": 19, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-petshop", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "petshop", "repetition": 2 }, "trialId": "petshop-gemma4-12b-q4-2026-10-29T22-37-28-516Z-0wvl", "success": true, "includedInCapability": true, "durationMs": 140168, "timing": { "startedAt": "2026-10-29T22:37:28.516Z", "finishedAt": "2026-10-29T22:39:48.684Z", "timeToFirstArtifactMs": 133987, "timeToLastArtifactWriteMs": 133991, "timeToFirstTokenMs": 12376, "firstTurnTtftMs": 10443, "timeToFirstToolCallMs": 21229 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 32870, "peakCpuPercent": 120.5, "firstAtMs": 0, "lastAtMs": 135478 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 105916, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44283, "totalOutputTokens": 3773, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 40478, "todayTokensCached": 39372, "todayTokensOut": 185, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 40478, "totalTokensCached": 39372, "totalTokensOut": 185, "totalCost": 0, "medianOutputTokensPerSec": 73.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 73.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7316, "cachedInputTokens": 7104, "outputTokens": 39, "cost": 0, "durationMs": 15608, "outputTokensPerSec": 71.65403921361577, "at": "2026-10-29T22:39:15.614Z" }, "lastUpdated": "2026-10-29T22:39:15.614Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:39:46.074Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136502, "meanTokensPerSec": 79.11, "genTokensPerSec": 79.11, "genTokens": 3773, "promptTokensPerSec": 1047.4, "peakRssMb": 32870, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 40478, "output": 185 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-petshop", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "petshop", "repetition": 2 }, "trialId": "petshop-mlx-qwen3-8-27b-q4-2026-10-29T22-40-37-084Z-s09e", "success": true, "includedInCapability": true, "durationMs": 496195, "timing": { "startedAt": "2026-10-29T22:40:37.085Z", "finishedAt": "2026-10-29T22:48:53.280Z", "timeToFirstArtifactMs": 212280, "timeToLastArtifactWriteMs": 355179, "timeToFirstTokenMs": 28767, "firstTurnTtftMs": 27075, "timeToFirstToolCallMs": 39845 }, "metrics": { "process": { "sampleCount": 99, "peakRssMb": 27622, "peakCpuPercent": 191, "firstAtMs": 0, "lastAtMs": 491989 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 99, "peakUsedMb": 98917, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 61663, "totalOutputTokens": 6927, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 60192, "todayTokensCached": 59346, "todayTokensOut": 1474, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 60192, "totalTokensCached": 59346, "totalTokensOut": 1474, "totalCost": 0, "medianOutputTokensPerSec": 25.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.3, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 19099, "cachedInputTokens": 18830, "outputTokens": 521, "cost": 0, "durationMs": 385713, "outputTokensPerSec": 16.487727744332794, "at": "2026-10-29T22:48:44.355Z" }, "lastUpdated": "2026-10-29T22:48:44.355Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T22:48:51.677Z" } }, "source": "engine-log" }, "derived": { "durationMs": 493518, "meanTokensPerSec": 21, "genTokensPerSec": 21, "genTokens": 6927, "promptTokensPerSec": null, "peakRssMb": 27622, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 60192, "output": 1474 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 397, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-petshop", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "petshop", "repetition": 2 }, "trialId": "petshop-qwen3-8-27b-q4-2026-10-29T22-49-42-388Z-96yq", "success": true, "includedInCapability": true, "durationMs": 1200735, "timing": { "startedAt": "2026-10-29T22:49:42.388Z", "finishedAt": "2026-10-29T23:09:43.123Z", "timeToFirstArtifactMs": 537357, "timeToLastArtifactWriteMs": 653122, "timeToFirstTokenMs": 23167, "firstTurnTtftMs": 21304, "timeToFirstToolCallMs": 29330 }, "metrics": { "process": { "sampleCount": 238, "peakRssMb": 57027, "peakCpuPercent": 114.7, "firstAtMs": 0, "lastAtMs": 1189318 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 238, "peakUsedMb": 128312, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 117181, "totalOutputTokens": 21237, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 94252, "todayTokensCached": 81185, "todayTokensOut": 2113, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 94252, "totalTokensCached": 81185, "totalTokensOut": 2113, "totalCost": 0, "medianOutputTokensPerSec": 23.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.4, "turns": 10 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9984, "cachedInputTokens": 9583, "outputTokens": 165, "cost": 0, "durationMs": 75254, "outputTokensPerSec": 25.44049282578102, "at": "2026-10-29T23:08:00.414Z" }, "lastUpdated": "2026-10-29T23:08:00.414Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:09:33.786Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1190342, "meanTokensPerSec": 27.17, "genTokensPerSec": 27.17, "genTokens": 21237, "promptTokensPerSec": 365.89, "peakRssMb": 57027, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 94252, "output": 2113 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-core-tankcombat", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "tankcombat", "repetition": 2 }, "trialId": "tankcombat-gemma4-12b-q4-2026-10-29T23-10-24-655Z-dset", "success": true, "includedInCapability": true, "durationMs": 67492, "timing": { "startedAt": "2026-10-29T23:10:24.656Z", "finishedAt": "2026-10-29T23:11:32.148Z", "timeToFirstArtifactMs": 58326, "timeToLastArtifactWriteMs": 58330, "timeToFirstTokenMs": 12243, "firstTurnTtftMs": 10525, "timeToFirstToolCallMs": 22232 }, "metrics": { "process": { "sampleCount": 13, "peakRssMb": 18232, "peakCpuPercent": 122.4, "firstAtMs": 0, "lastAtMs": 60214 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 81651, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13561, "totalOutputTokens": 2998, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7397, "todayTokensCached": 7225, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7397, "totalTokensCached": 7225, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 64.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 64.8, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7397, "cachedInputTokens": 7225, "outputTokens": 23, "cost": 0, "durationMs": 22306, "outputTokensPerSec": 64.77847234695349, "at": "2026-10-29T23:10:48.680Z" }, "lastUpdated": "2026-10-29T23:10:48.680Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:11:28.113Z" } }, "source": "engine-log" }, "derived": { "durationMs": 62513, "meanTokensPerSec": 86.4, "genTokensPerSec": 86.4, "genTokens": 2998, "promptTokensPerSec": 1077.07, "peakRssMb": 18232, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7397, "output": 23 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-tankcombat", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "tankcombat", "repetition": 2 }, "trialId": "tankcombat-mlx-qwen3-8-27b-q4-2026-10-29T23-12-19-097Z-vvpm", "success": true, "includedInCapability": true, "durationMs": 131324, "timing": { "startedAt": "2026-10-29T23:12:19.098Z", "finishedAt": "2026-10-29T23:14:30.422Z", "timeToFirstArtifactMs": 125175, "timeToLastArtifactWriteMs": 125180, "timeToFirstTokenMs": 22916, "firstTurnTtftMs": 21242, "timeToFirstToolCallMs": 31994 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 18300, "peakCpuPercent": 116.9, "firstAtMs": 0, "lastAtMs": 125475 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 93270, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17235, "totalOutputTokens": 4059, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 17219, "todayTokensCached": 11834, "todayTokensOut": 4000, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 17219, "totalTokensCached": 11834, "totalTokensOut": 4000, "totalCost": 0, "medianOutputTokensPerSec": 43.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 43.9, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5220, "cachedInputTokens": 0, "outputTokens": 3860, "cost": 0, "durationMs": 88584, "outputTokensPerSec": 47.46509708848556, "at": "2026-10-29T23:14:24.278Z" }, "lastUpdated": "2026-10-29T23:14:24.278Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:14:27.947Z" } }, "source": "engine-log" }, "derived": { "durationMs": 127780, "meanTokensPerSec": 49, "genTokensPerSec": 49, "genTokens": 4059, "promptTokensPerSec": null, "peakRssMb": 18300, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 17219, "output": 4000 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 775, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-tankcombat", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "tankcombat", "repetition": 2 }, "trialId": "tankcombat-qwen3-8-27b-q4-2026-10-29T23-15-21-924Z-1kb3", "success": true, "includedInCapability": true, "durationMs": 201226, "timing": { "startedAt": "2026-10-29T23:15:21.924Z", "finishedAt": "2026-10-29T23:18:43.150Z", "timeToFirstArtifactMs": 193449, "timeToLastArtifactWriteMs": 193454, "timeToFirstTokenMs": 22471, "firstTurnTtftMs": 20616, "timeToFirstToolCallMs": 34897 }, "metrics": { "process": { "sampleCount": 40, "peakRssMb": 39606, "peakCpuPercent": 118.5, "firstAtMs": 0, "lastAtMs": 195698 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 101522, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13806, "totalOutputTokens": 5594, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15554, "todayTokensCached": 14290, "todayTokensOut": 2260, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15554, "totalTokensCached": 14290, "totalTokensOut": 2260, "totalCost": 0, "medianOutputTokensPerSec": 31.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.9, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7850, "cachedInputTokens": 7145, "outputTokens": 2154, "cost": 0, "durationMs": 68577, "outputTokensPerSec": 32.36708003070979, "at": "2026-10-29T23:17:10.247Z" }, "lastUpdated": "2026-10-29T23:17:10.247Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:18:40.951Z" } }, "source": "engine-log" }, "derived": { "durationMs": 197978, "meanTokensPerSec": 36.21, "genTokensPerSec": 36.21, "genTokens": 5594, "promptTokensPerSec": 484.75, "peakRssMb": 39606, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15554, "output": 2260 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-tankcombat", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "tankcombat", "repetition": 2 }, "trialId": "tankcombat-mlx-gemma4-12b-q4-2026-10-29T23-19-31-332Z-4q3x", "success": true, "includedInCapability": true, "durationMs": 111377, "timing": { "startedAt": "2026-10-29T23:19:31.333Z", "finishedAt": "2026-10-29T23:21:22.710Z", "timeToFirstArtifactMs": 105598, "timeToLastArtifactWriteMs": 105610, "timeToFirstTokenMs": 18179, "firstTurnTtftMs": 16485, "timeToFirstToolCallMs": 39973 }, "metrics": { "process": { "sampleCount": 22, "peakRssMb": 11234, "peakCpuPercent": 121.7, "firstAtMs": 0, "lastAtMs": 105530 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 22, "peakUsedMb": 77248, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16747, "totalOutputTokens": 2375, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 16715, "todayTokensCached": 10421, "todayTokensOut": 1871, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 16715, "totalTokensCached": 10421, "totalTokensOut": 1871, "totalCost": 0, "medianOutputTokensPerSec": 36.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6115, "cachedInputTokens": 0, "outputTokens": 1848, "cost": 0, "durationMs": 59028, "outputTokensPerSec": 33.97841488398039, "at": "2026-10-29T23:21:16.943Z" }, "lastUpdated": "2026-10-29T23:21:16.943Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:21:20.214Z" } }, "source": "engine-log" }, "derived": { "durationMs": 107817, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 2375, "promptTokensPerSec": null, "peakRssMb": 11234, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 16715, "output": 1871 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1184.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-core-schema-migration", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "schema-migration", "repetition": 2 }, "trialId": "schema-migration-mlx-qwen3-8-27b-q4-2026-10-29T23-22-10-027Z-woe1", "success": true, "includedInCapability": true, "durationMs": 224208, "timing": { "startedAt": "2026-10-29T23:22:10.027Z", "finishedAt": "2026-10-29T23:25:54.235Z", "timeToFirstArtifactMs": 28282, "timeToLastArtifactWriteMs": 219689, "timeToFirstTokenMs": 15213, "firstTurnTtftMs": 13564, "timeToFirstToolCallMs": 19805 }, "metrics": { "process": { "sampleCount": 45, "peakRssMb": 19025, "peakCpuPercent": 110.2, "firstAtMs": 0, "lastAtMs": 220858 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 45, "peakUsedMb": 94521, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 42461, "totalOutputTokens": 3035, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 28208, "todayTokensCached": 3419, "todayTokensOut": 2474, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 28208, "totalTokensCached": 3419, "totalTokensOut": 2474, "totalCost": 0, "medianOutputTokensPerSec": 40.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 40.3, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 6970, "cachedInputTokens": 0, "outputTokens": 1027, "cost": 0, "durationMs": 47497, "outputTokensPerSec": 29.2011930047102, "at": "2026-10-29T23:25:49.716Z" }, "lastUpdated": "2026-10-29T23:25:49.716Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:25:52.975Z" } }, "source": "engine-log" }, "derived": { "durationMs": 221890, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 3035, "promptTokensPerSec": null, "peakRssMb": 19025, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 28208, "output": 2474 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 657, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-schema-migration", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "schema-migration", "repetition": 2 }, "trialId": "schema-migration-qwen3-8-27b-q4-2026-10-29T23-26-16-399Z-omfz", "success": true, "includedInCapability": true, "durationMs": 255944, "timing": { "startedAt": "2026-10-29T23:26:16.399Z", "finishedAt": "2026-10-29T23:30:32.343Z", "timeToFirstArtifactMs": 40827, "timeToLastArtifactWriteMs": 252627, "timeToFirstTokenMs": 19633, "firstTurnTtftMs": 17811, "timeToFirstToolCallMs": 21707 }, "metrics": { "process": { "sampleCount": 51, "peakRssMb": 46047, "peakCpuPercent": 119.1, "firstAtMs": 0, "lastAtMs": 250997 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 51, "peakUsedMb": 104625, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 49187, "totalOutputTokens": 3022, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 57754, "todayTokensCached": 15747, "todayTokensOut": 2878, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 57754, "totalTokensCached": 15747, "totalTokensOut": 2878, "totalCost": 0, "medianOutputTokensPerSec": 34.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 34.6, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11307, "cachedInputTokens": 10490, "outputTokens": 978, "cost": 0, "durationMs": 40035, "outputTokensPerSec": 26.23741503723457, "at": "2026-10-29T23:30:29.026Z" }, "lastUpdated": "2026-10-29T23:30:29.026Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:30:29.721Z" } }, "source": "engine-log" }, "derived": { "durationMs": 252282, "meanTokensPerSec": 32.09, "genTokensPerSec": 32.09, "genTokens": 3022, "promptTokensPerSec": 379.46, "peakRssMb": 46047, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 57754, "output": 2878 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-schema-migration", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "schema-migration", "repetition": 2 }, "trialId": "schema-migration-mlx-gemma4-12b-q4-2026-10-29T23-30-55-714Z-au6z", "success": true, "includedInCapability": true, "durationMs": 279819, "timing": { "startedAt": "2026-10-29T23:30:55.715Z", "finishedAt": "2026-10-29T23:35:35.534Z", "timeToFirstArtifactMs": 36086, "timeToLastArtifactWriteMs": 274223, "timeToFirstTokenMs": 13492, "firstTurnTtftMs": 11833, "timeToFirstToolCallMs": 24540 }, "metrics": { "process": { "sampleCount": 56, "peakRssMb": 10664, "peakCpuPercent": 115.8, "firstAtMs": 0, "lastAtMs": 276280 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 56, "peakUsedMb": 77598, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 80075, "totalOutputTokens": 4158, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 91683, "todayTokensCached": 38404, "todayTokensOut": 2239, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 91683, "totalTokensCached": 38404, "totalTokensOut": 2239, "totalCost": 0, "medianOutputTokensPerSec": 33, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 33, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11847, "cachedInputTokens": 1252, "outputTokens": 273, "cost": 0, "durationMs": 20280, "outputTokensPerSec": 32.18015317394874, "at": "2026-10-29T23:35:29.938Z" }, "lastUpdated": "2026-10-29T23:35:29.938Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:35:34.352Z" } }, "source": "engine-log" }, "derived": { "durationMs": 277574, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 4158, "promptTokensPerSec": null, "peakRssMb": 10664, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 91683, "output": 2239 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 925.5, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-schema-migration", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "schema-migration", "repetition": 2 }, "trialId": "schema-migration-gemma4-12b-q4-2026-10-29T23-35-58-504Z-fx11", "success": true, "includedInCapability": true, "durationMs": 190564, "timing": { "startedAt": "2026-10-29T23:35:58.505Z", "finishedAt": "2026-10-29T23:39:09.069Z", "timeToFirstArtifactMs": 29404, "timeToLastArtifactWriteMs": 186933, "timeToFirstTokenMs": 11283, "firstTurnTtftMs": 9447, "timeToFirstToolCallMs": 16887 }, "metrics": { "process": { "sampleCount": 38, "peakRssMb": 25947, "peakCpuPercent": 118.9, "firstAtMs": 0, "lastAtMs": 185693 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 38, "peakUsedMb": 83391, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 58551, "totalOutputTokens": 6358, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 44841, "todayTokensCached": 19698, "todayTokensOut": 3873, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 44841, "totalTokensCached": 19698, "totalTokensOut": 3873, "totalCost": 0, "medianOutputTokensPerSec": 79.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 79.7, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10563, "outputTokens": 369, "cost": 0, "durationMs": 33321, "outputTokensPerSec": 89.67248343800401, "at": "2026-10-29T23:38:23.664Z" }, "lastUpdated": "2026-10-29T23:38:23.664Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:39:06.264Z" } }, "source": "engine-log" }, "derived": { "durationMs": 186718, "meanTokensPerSec": 70.78, "genTokensPerSec": 70.78, "genTokens": 6358, "promptTokensPerSec": 910.26, "peakRssMb": 25947, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44841, "output": 3873 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-core-failing-tests-spec", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "failing-tests-spec", "repetition": 2 }, "trialId": "failing-tests-spec-qwen3-8-27b-q4-2026-10-29T23-39-30-396Z-21n6", "success": true, "includedInCapability": true, "durationMs": 53944, "timing": { "startedAt": "2026-10-29T23:39:30.397Z", "finishedAt": "2026-10-29T23:40:24.341Z", "timeToFirstArtifactMs": 47976, "timeToLastArtifactWriteMs": 47978, "timeToFirstTokenMs": 18238, "firstTurnTtftMs": 16473, "timeToFirstToolCallMs": 19397 }, "metrics": { "process": { "sampleCount": 11, "peakRssMb": 37702, "peakCpuPercent": 116.6, "firstAtMs": 0, "lastAtMs": 50171 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 11, "peakUsedMb": 98183, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8648, "totalOutputTokens": 622, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4964, "todayTokensCached": 0, "todayTokensOut": 565, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4964, "totalTokensCached": 0, "totalTokensOut": 565, "totalCost": 0, "medianOutputTokensPerSec": 36, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4964, "outputTokens": 565, "cost": 0, "durationMs": 46213, "outputTokensPerSec": 35.955519197697214, "at": "2026-10-29T23:40:18.375Z" }, "lastUpdated": "2026-10-29T23:40:18.375Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:40:23.154Z" } }, "source": "engine-log" }, "derived": { "durationMs": 51714, "meanTokensPerSec": 36.53, "genTokensPerSec": 36.53, "genTokens": 622, "promptTokensPerSec": 384.91, "peakRssMb": 37702, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4964, "output": 565 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-failing-tests-spec", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "failing-tests-spec", "repetition": 2 }, "trialId": "failing-tests-spec-mlx-gemma4-12b-q4-2026-10-29T23-40-45-443Z-x030", "success": false, "includedInCapability": true, "durationMs": 240293, "timing": { "startedAt": "2026-10-29T23:40:45.444Z", "finishedAt": "2026-10-29T23:44:45.737Z", "timeToFirstArtifactMs": 77385, "timeToLastArtifactWriteMs": 237972, "timeToFirstTokenMs": 12345, "firstTurnTtftMs": 10705, "timeToFirstToolCallMs": 12902 }, "metrics": { "process": { "sampleCount": 48, "peakRssMb": 10766, "peakCpuPercent": 120.1, "firstAtMs": 0, "lastAtMs": 235978 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 48, "peakUsedMb": 79127, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 93953, "totalOutputTokens": 3308, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 52925, "todayTokensCached": 33959, "todayTokensOut": 2296, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 52925, "totalTokensCached": 33959, "totalTokensOut": 2296, "totalCost": 0, "medianOutputTokensPerSec": 33.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 33.2, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17525, "cachedInputTokens": 16420, "outputTokens": 581, "cost": 0, "durationMs": 19390, "outputTokensPerSec": 31.423928311410275, "at": "2026-10-29T23:44:43.416Z" }, "lastUpdated": "2026-10-29T23:44:43.416Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:44:44.523Z" } }, "source": "engine-log" }, "derived": { "durationMs": 238025, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 3308, "promptTokensPerSec": null, "peakRssMb": 10766, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 52925, "output": 2296 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1146, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-failing-tests-spec", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "failing-tests-spec", "repetition": 2 }, "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-29T23-45-08-198Z-4ypl", "success": true, "includedInCapability": true, "durationMs": 39104, "timing": { "startedAt": "2026-10-29T23:45:08.199Z", "finishedAt": "2026-10-29T23:45:47.303Z", "timeToFirstArtifactMs": 31776, "timeToLastArtifactWriteMs": 31780, "timeToFirstTokenMs": 11102, "firstTurnTtftMs": 9235, "timeToFirstToolCallMs": 12466 }, "metrics": { "process": { "sampleCount": 8, "peakRssMb": 16578, "peakCpuPercent": 86.7, "firstAtMs": 0, "lastAtMs": 35147 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 75007, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13667, "totalOutputTokens": 805, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6622, "todayTokensCached": 0, "todayTokensOut": 661, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6622, "totalTokensCached": 0, "totalTokensOut": 661, "totalCost": 0, "medianOutputTokensPerSec": 75.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 75.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6622, "outputTokens": 661, "cost": 0, "durationMs": 29912, "outputTokensPerSec": 75.90365020653843, "at": "2026-10-29T23:45:39.979Z" }, "lastUpdated": "2026-10-29T23:45:39.979Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:45:45.694Z" } }, "source": "engine-log" }, "derived": { "durationMs": 36444, "meanTokensPerSec": 78.33, "genTokensPerSec": 78.33, "genTokens": 805, "promptTokensPerSec": 864.88, "peakRssMb": 16578, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6622, "output": 661 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-failing-tests-spec", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "failing-tests-spec", "repetition": 2 }, "trialId": "failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-29T23-46-07-815Z-uxrq", "success": true, "includedInCapability": true, "durationMs": 43466, "timing": { "startedAt": "2026-10-29T23:46:07.816Z", "finishedAt": "2026-10-29T23:46:51.282Z", "timeToFirstArtifactMs": 39587, "timeToLastArtifactWriteMs": 39587, "timeToFirstTokenMs": 15238, "firstTurnTtftMs": 13570, "timeToFirstToolCallMs": 17193 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 17973, "peakCpuPercent": 118.9, "firstAtMs": 1, "lastAtMs": 40152 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 95550, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8480, "totalOutputTokens": 372, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4918, "todayTokensCached": 0, "todayTokensOut": 449, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4918, "totalTokensCached": 0, "totalTokensOut": 449, "totalCost": 0, "medianOutputTokensPerSec": 36.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.3, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4918, "cachedInputTokens": 0, "outputTokens": 449, "cost": 0, "durationMs": 37926, "outputTokensPerSec": 36.326342522610744, "at": "2026-10-29T23:46:47.410Z" }, "lastUpdated": "2026-10-29T23:46:47.410Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:46:50.335Z" } }, "source": "engine-log" }, "derived": { "durationMs": 41451, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 372, "promptTokensPerSec": null, "peakRssMb": 17973, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 4918, "output": 449 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 516, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-core-symptom-debug", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "symptom-debug", "repetition": 2 }, "trialId": "symptom-debug-mlx-gemma4-12b-q4-2026-10-29T23-47-13-425Z-2hng", "success": true, "includedInCapability": true, "durationMs": 349492, "timing": { "startedAt": "2026-10-29T23:47:13.426Z", "finishedAt": "2026-10-29T23:53:02.918Z", "timeToFirstArtifactMs": 343405, "timeToLastArtifactWriteMs": 343405, "timeToFirstTokenMs": 12131, "firstTurnTtftMs": 10498, "timeToFirstToolCallMs": 12663 }, "metrics": { "process": { "sampleCount": 70, "peakRssMb": 10421, "peakCpuPercent": 111.1, "firstAtMs": 0, "lastAtMs": 346671 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 70, "peakUsedMb": 77203, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6931, "totalOutputTokens": 11332, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 13324, "todayTokensCached": 12938, "todayTokensOut": 8296, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 13324, "totalTokensCached": 12938, "totalTokensOut": 8296, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6851, "cachedInputTokens": 6575, "outputTokens": 104, "cost": 0, "durationMs": 107147, "outputTokensPerSec": 35.172203069674815, "at": "2026-10-29T23:53:00.442Z" }, "lastUpdated": "2026-10-29T23:53:00.442Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:53:01.699Z" } }, "source": "engine-log" }, "derived": { "durationMs": 347207, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 11332, "promptTokensPerSec": null, "peakRssMb": 10421, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 13324, "output": 8296 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 749.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-symptom-debug", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "symptom-debug", "repetition": 2 }, "trialId": "symptom-debug-gemma4-12b-q4-2026-10-29T23-53-25-309Z-smbm", "success": true, "includedInCapability": true, "durationMs": 175375, "timing": { "startedAt": "2026-10-29T23:53:25.309Z", "finishedAt": "2026-10-29T23:56:20.684Z", "timeToFirstArtifactMs": 165903, "timeToLastArtifactWriteMs": 165903, "timeToFirstTokenMs": 11131, "firstTurnTtftMs": 9254, "timeToFirstToolCallMs": 16443 }, "metrics": { "process": { "sampleCount": 34, "peakRssMb": 19483, "peakCpuPercent": 99.5, "firstAtMs": 0, "lastAtMs": 165593 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 76087, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5936, "totalOutputTokens": 1954, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:56:12.475Z" } }, "source": "engine-log" }, "derived": { "durationMs": 166117, "meanTokensPerSec": 67.19, "genTokensPerSec": 67.19, "genTokens": 1954, "promptTokensPerSec": 775.33, "peakRssMb": 19483, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-symptom-debug", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "symptom-debug", "repetition": 2 }, "trialId": "symptom-debug-mlx-qwen3-8-27b-q4-2026-10-29T23-56-42-218Z-a631", "success": true, "includedInCapability": true, "durationMs": 47633, "timing": { "startedAt": "2026-10-29T23:56:42.218Z", "finishedAt": "2026-10-29T23:57:29.851Z", "timeToFirstArtifactMs": 41560, "timeToLastArtifactWriteMs": 41560, "timeToFirstTokenMs": 16253, "firstTurnTtftMs": 14603, "timeToFirstToolCallMs": 18027 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 17966, "peakCpuPercent": 122.4, "firstAtMs": 0, "lastAtMs": 45167 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 93324, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5567, "totalOutputTokens": 414, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:57:28.980Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45691, "meanTokensPerSec": 27, "genTokensPerSec": 27, "genTokens": 414, "promptTokensPerSec": null, "peakRssMb": 17966, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 512, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-symptom-debug", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "symptom-debug", "repetition": 2 }, "trialId": "symptom-debug-qwen3-8-27b-q4-2026-10-29T23-57-51-588Z-zlal", "success": true, "includedInCapability": true, "durationMs": 43461, "timing": { "startedAt": "2026-10-29T23:57:51.589Z", "finishedAt": "2026-10-29T23:58:35.050Z", "timeToFirstArtifactMs": 40415, "timeToLastArtifactWriteMs": 40415, "timeToFirstTokenMs": 17735, "firstTurnTtftMs": 15915, "timeToFirstToolCallMs": 18971 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 37178, "peakCpuPercent": 118, "firstAtMs": 0, "lastAtMs": 40146 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 97447, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5482, "totalOutputTokens": 589, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:58:33.306Z" } }, "source": "engine-log" }, "derived": { "durationMs": 40670, "meanTokensPerSec": 31.24, "genTokensPerSec": 31.24, "genTokens": 589, "promptTokensPerSec": 419.24, "peakRssMb": 37178, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-core-data-wrangle", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "data-wrangle", "repetition": 2 }, "trialId": "data-wrangle-gemma4-12b-q4-2026-10-29T23-58-56-992Z-ewhc", "success": true, "includedInCapability": true, "durationMs": 61862, "timing": { "startedAt": "2026-10-29T23:58:56.993Z", "finishedAt": "2026-10-29T23:59:58.855Z", "timeToFirstArtifactMs": 33018, "timeToLastArtifactWriteMs": 51210, "timeToFirstTokenMs": 10319, "firstTurnTtftMs": 8513, "timeToFirstToolCallMs": 16701 }, "metrics": { "process": { "sampleCount": 11, "peakRssMb": 19393, "peakCpuPercent": 112.7, "firstAtMs": 0, "lastAtMs": 50181 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 11, "peakUsedMb": 76489, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 9982, "totalOutputTokens": 2972, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 9949, "todayTokensCached": 9893, "todayTokensOut": 232, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 9949, "totalTokensCached": 9893, "totalTokensOut": 232, "totalCost": 0, "medianOutputTokensPerSec": 72.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 72.1, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9949, "cachedInputTokens": 9893, "outputTokens": 232, "cost": 0, "durationMs": 49404, "outputTokensPerSec": 72.09663570880825, "at": "2026-10-29T23:59:48.203Z" }, "lastUpdated": "2026-10-29T23:59:48.203Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-29T23:59:48.751Z" } }, "source": "engine-log" }, "derived": { "durationMs": 50708, "meanTokensPerSec": 86.99, "genTokensPerSec": 86.99, "genTokens": 2972, "promptTokensPerSec": 898.05, "peakRssMb": 19393, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 9949, "output": 232 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-data-wrangle", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "data-wrangle", "repetition": 2 }, "trialId": "data-wrangle-mlx-qwen3-8-27b-q4-2026-10-30T00-00-20-916Z-fib5", "success": true, "includedInCapability": true, "durationMs": 72914, "timing": { "startedAt": "2026-10-30T00:00:20.917Z", "finishedAt": "2026-10-30T00:01:33.831Z", "timeToFirstArtifactMs": 58217, "timeToLastArtifactWriteMs": 58217, "timeToFirstTokenMs": 15414, "firstTurnTtftMs": 13763, "timeToFirstToolCallMs": 16865 }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 18023, "peakCpuPercent": 120, "firstAtMs": 0, "lastAtMs": 70254 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 93913, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 7334, "totalOutputTokens": 1238, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:01:32.761Z" } }, "source": "engine-log" }, "derived": { "durationMs": 70775, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 1238, "promptTokensPerSec": null, "peakRssMb": 18023, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 612, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-data-wrangle", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "data-wrangle", "repetition": 2 }, "trialId": "data-wrangle-qwen3-8-27b-q4-2026-10-30T00-01-55-563Z-hmlm", "success": true, "includedInCapability": true, "durationMs": 263124, "timing": { "startedAt": "2026-10-30T00:01:55.564Z", "finishedAt": "2026-10-30T00:06:18.688Z", "timeToFirstArtifactMs": 64592, "timeToLastArtifactWriteMs": 239183, "timeToFirstTokenMs": 17588, "firstTurnTtftMs": 15765, "timeToFirstToolCallMs": 19666 }, "metrics": { "process": { "sampleCount": 52, "peakRssMb": 39634, "peakCpuPercent": 118.5, "firstAtMs": 0, "lastAtMs": 255818 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 52, "peakUsedMb": 88532, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 11776, "totalOutputTokens": 5124, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:06:12.948Z" } }, "source": "engine-log" }, "derived": { "durationMs": 256340, "meanTokensPerSec": 24.14, "genTokensPerSec": 24.14, "genTokens": 5124, "promptTokensPerSec": 349.36, "peakRssMb": 39634, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-data-wrangle", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "data-wrangle", "repetition": 2 }, "trialId": "data-wrangle-mlx-gemma4-12b-q4-2026-10-30T00-06-40-900Z-cjrj", "success": false, "includedInCapability": true, "durationMs": 575767, "timing": { "startedAt": "2026-10-30T00:06:40.900Z", "finishedAt": "2026-10-30T00:16:16.667Z", "timeToFirstArtifactMs": 81349, "timeToLastArtifactWriteMs": 570510, "timeToFirstTokenMs": 13859, "firstTurnTtftMs": 12227, "timeToFirstToolCallMs": 35242 }, "metrics": { "process": { "sampleCount": 115, "peakRssMb": 10650, "peakCpuPercent": 102.1, "firstAtMs": 0, "lastAtMs": 572800 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 115, "peakUsedMb": 67553, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 59495, "totalOutputTokens": 15049, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 66708, "todayTokensCached": 19309, "todayTokensOut": 11374, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 66708, "totalTokensCached": 19309, "totalTokensOut": 11374, "totalCost": 0, "medianOutputTokensPerSec": 32.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 32.2, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18858, "cachedInputTokens": 0, "outputTokens": 1023, "cost": 0, "durationMs": 51233, "outputTokensPerSec": 32.08378627414882, "at": "2026-10-30T00:16:11.410Z" }, "lastUpdated": "2026-10-30T00:16:11.410Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:16:15.308Z" } }, "source": "engine-log" }, "derived": { "durationMs": 573343, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 15049, "promptTokensPerSec": null, "peakRssMb": 10650, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 66708, "output": 11374 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 983, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-core-incident-postmortem", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "incident-postmortem", "repetition": 2 }, "trialId": "incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-30T00-16-38-042Z-e5em", "success": true, "includedInCapability": true, "durationMs": 931079, "timing": { "startedAt": "2026-10-30T00:16:38.042Z", "finishedAt": "2026-10-30T00:32:09.121Z", "timeToFirstArtifactMs": 845480, "timeToLastArtifactWriteMs": 924741, "timeToFirstTokenMs": 29686, "firstTurnTtftMs": 28040, "timeToFirstToolCallMs": 34131 }, "metrics": { "process": { "sampleCount": 186, "peakRssMb": 18196, "peakCpuPercent": 116.6, "firstAtMs": 0, "lastAtMs": 928103 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 186, "peakUsedMb": 85037, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 42531, "totalOutputTokens": 18180, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 31894, "todayTokensCached": 17072, "todayTokensOut": 441, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 31894, "totalTokensCached": 17072, "totalTokensOut": 441, "totalCost": 0, "medianOutputTokensPerSec": 23.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.4, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14746, "cachedInputTokens": 0, "outputTokens": 189, "cost": 0, "durationMs": 61208, "outputTokensPerSec": 27.162733633148413, "at": "2026-10-30T00:32:02.787Z" }, "lastUpdated": "2026-10-30T00:32:02.787Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:32:07.755Z" } }, "source": "engine-log" }, "derived": { "durationMs": 928649, "meanTokensPerSec": 23, "genTokensPerSec": 23, "genTokens": 18180, "promptTokensPerSec": null, "peakRssMb": 18196, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 31894, "output": 441 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 507.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-incident-postmortem", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "incident-postmortem", "repetition": 2 }, "trialId": "incident-postmortem-qwen3-8-27b-q4-2026-10-30T00-32-31-792Z-29is", "success": true, "includedInCapability": true, "durationMs": 521965, "timing": { "startedAt": "2026-10-30T00:32:31.793Z", "finishedAt": "2026-10-30T00:41:13.758Z", "timeToFirstArtifactMs": 513576, "timeToLastArtifactWriteMs": 513576, "timeToFirstTokenMs": 36407, "firstTurnTtftMs": 34538, "timeToFirstToolCallMs": 40980 }, "metrics": { "process": { "sampleCount": 104, "peakRssMb": 40485, "peakCpuPercent": 117.1, "firstAtMs": 0, "lastAtMs": 516684 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 104, "peakUsedMb": 89680, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 30988, "totalOutputTokens": 10814, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:41:10.051Z" } }, "source": "engine-log" }, "derived": { "durationMs": 517206, "meanTokensPerSec": 25.45, "genTokensPerSec": 25.45, "genTokens": 10814, "promptTokensPerSec": 372.88, "peakRssMb": 40485, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-incident-postmortem", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "incident-postmortem", "repetition": 2 }, "trialId": "incident-postmortem-mlx-gemma4-12b-q4-2026-10-30T00-41-35-480Z-a2in", "success": false, "includedInCapability": true, "durationMs": 183317, "timing": { "startedAt": "2026-10-30T00:41:35.480Z", "finishedAt": "2026-10-30T00:44:38.797Z", "timeToFirstArtifactMs": 117686, "timeToLastArtifactWriteMs": 178014, "timeToFirstTokenMs": 18304, "firstTurnTtftMs": 16668, "timeToFirstToolCallMs": 40958 }, "metrics": { "process": { "sampleCount": 37, "peakRssMb": 10442, "peakCpuPercent": 113.8, "firstAtMs": 1, "lastAtMs": 180703 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 37, "peakUsedMb": 68586, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 35193, "totalOutputTokens": 3798, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 79654, "todayTokensCached": 76936, "todayTokensOut": 942, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 79654, "totalTokensCached": 76936, "totalTokensOut": 942, "totalCost": 0, "medianOutputTokensPerSec": 34.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.6, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17374, "cachedInputTokens": 16205, "outputTokens": 182, "cost": 0, "durationMs": 6181, "outputTokensPerSec": 34.836795268548464, "at": "2026-10-30T00:44:33.497Z" }, "lastUpdated": "2026-10-30T00:44:33.497Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:44:37.792Z" } }, "source": "engine-log" }, "derived": { "durationMs": 181245, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 3798, "promptTokensPerSec": null, "peakRssMb": 10442, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 79654, "output": 942 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1090, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-incident-postmortem", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "incident-postmortem", "repetition": 2 }, "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-30T00-45-00-527Z-lw6g", "success": true, "includedInCapability": true, "durationMs": 145051, "timing": { "startedAt": "2026-10-30T00:45:00.528Z", "finishedAt": "2026-10-30T00:47:25.579Z", "timeToFirstArtifactMs": 97526, "timeToLastArtifactWriteMs": 141331, "timeToFirstTokenMs": 15873, "firstTurnTtftMs": 14010, "timeToFirstToolCallMs": 24544 }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 20917, "peakCpuPercent": 96.7, "firstAtMs": 0, "lastAtMs": 140496 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 69490, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44489, "totalOutputTokens": 5170, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 31926, "todayTokensCached": 26950, "todayTokensOut": 661, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 31926, "totalTokensCached": 26950, "totalTokensOut": 661, "totalCost": 0, "medianOutputTokensPerSec": 57.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 57.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12510, "cachedInputTokens": 8897, "outputTokens": 537, "cost": 0, "durationMs": 34321, "outputTokensPerSec": 54.214505151541175, "at": "2026-10-30T00:47:21.859Z" }, "lastUpdated": "2026-10-30T00:47:21.859Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:47:22.618Z" } }, "source": "engine-log" }, "derived": { "durationMs": 141039, "meanTokensPerSec": 61.73, "genTokensPerSec": 61.73, "genTokens": 5170, "promptTokensPerSec": 970.52, "peakRssMb": 20917, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 31926, "output": 661 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-core-ops-runbook-anomaly", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 2 }, "trialId": "ops-runbook-anomaly-qwen3-8-27b-q4-2026-10-30T00-47-47-730Z-oicm", "success": true, "includedInCapability": true, "durationMs": 118626, "timing": { "startedAt": "2026-10-30T00:47:47.730Z", "finishedAt": "2026-10-30T00:49:46.356Z", "timeToFirstArtifactMs": 47891, "timeToLastArtifactWriteMs": 114122, "timeToFirstTokenMs": 35034, "firstTurnTtftMs": 33192, "timeToFirstToolCallMs": 37669 }, "metrics": { "process": { "sampleCount": 24, "peakRssMb": 40572, "peakCpuPercent": 119.1, "firstAtMs": 0, "lastAtMs": 115393 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 89824, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 22908, "totalOutputTokens": 1466, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18267, "todayTokensCached": 12431, "todayTokensOut": 440, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18267, "totalTokensCached": 12431, "totalTokensOut": 440, "totalCost": 0, "medianOutputTokensPerSec": 35.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 35.6, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5584, "outputTokens": 97, "cost": 0, "durationMs": 27114, "outputTokensPerSec": 43.29543179075318, "at": "2026-10-30T00:49:41.852Z" }, "lastUpdated": "2026-10-30T00:49:41.852Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:49:44.715Z" } }, "source": "engine-log" }, "derived": { "durationMs": 115937, "meanTokensPerSec": 31.81, "genTokensPerSec": 31.81, "genTokens": 1466, "promptTokensPerSec": 407.72, "peakRssMb": 40572, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18267, "output": 440 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-ops-runbook-anomaly", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 2 }, "trialId": "ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-30T00-50-27-551Z-iukg", "success": true, "includedInCapability": true, "durationMs": 434887, "timing": { "startedAt": "2026-10-30T00:50:27.551Z", "finishedAt": "2026-10-30T00:57:42.438Z", "timeToFirstArtifactMs": 22680, "timeToLastArtifactWriteMs": 430411, "timeToFirstTokenMs": 14601, "firstTurnTtftMs": 12878, "timeToFirstToolCallMs": 20097 }, "metrics": { "process": { "sampleCount": 87, "peakRssMb": 10548, "peakCpuPercent": 115.3, "firstAtMs": 0, "lastAtMs": 431861 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 87, "peakUsedMb": 67309, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 105113, "totalOutputTokens": 8851, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 59645, "todayTokensCached": 31633, "todayTokensOut": 8777, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 59645, "totalTokensCached": 31633, "totalTokensOut": 8777, "totalCost": 0, "medianOutputTokensPerSec": 34.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.6, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11356, "cachedInputTokens": 1201, "outputTokens": 113, "cost": 0, "durationMs": 42501, "outputTokensPerSec": 34.80450833227215, "at": "2026-10-30T00:57:37.962Z" }, "lastUpdated": "2026-10-30T00:57:37.962Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:57:41.107Z" } }, "source": "engine-log" }, "derived": { "durationMs": 432402, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 8851, "promptTokensPerSec": null, "peakRssMb": 10548, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 59645, "output": 8777 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 989.5, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-ops-runbook-anomaly", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 2 }, "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-30T00-58-04-249Z-4cwv", "success": true, "includedInCapability": true, "durationMs": 83903, "timing": { "startedAt": "2026-10-30T00:58:04.249Z", "finishedAt": "2026-10-30T00:59:28.152Z", "timeToFirstArtifactMs": 22286, "timeToLastArtifactWriteMs": 81355, "timeToFirstTokenMs": 15895, "firstTurnTtftMs": 14031, "timeToFirstToolCallMs": 20401 }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 28050, "peakCpuPercent": 116.2, "firstAtMs": 0, "lastAtMs": 80304 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 76385, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 54675, "totalOutputTokens": 1108, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 26816, "todayTokensCached": 11348, "todayTokensOut": 298, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 26816, "totalTokensCached": 11348, "totalTokensOut": 298, "totalCost": 0, "medianOutputTokensPerSec": 88.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88.9, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8288, "outputTokens": 139, "cost": 0, "durationMs": 24930, "outputTokensPerSec": 88.87801170871197, "at": "2026-10-30T00:59:25.604Z" }, "lastUpdated": "2026-10-30T00:59:25.604Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T00:59:26.127Z" } }, "source": "engine-log" }, "derived": { "durationMs": 80830, "meanTokensPerSec": 79.58, "genTokensPerSec": 79.58, "genTokens": 1108, "promptTokensPerSec": 978.6, "peakRssMb": 28050, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 26816, "output": 298 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-ops-runbook-anomaly", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 2 }, "trialId": "ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-30T00-59-49-741Z-neo6", "success": true, "includedInCapability": true, "durationMs": 123302, "timing": { "startedAt": "2026-10-30T00:59:49.742Z", "finishedAt": "2026-10-30T01:01:53.044Z", "timeToFirstArtifactMs": 109601, "timeToLastArtifactWriteMs": 118382, "timeToFirstTokenMs": 28542, "firstTurnTtftMs": 26884, "timeToFirstToolCallMs": 29616 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 17873, "peakCpuPercent": 131.7, "firstAtMs": 1, "lastAtMs": 120404 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 84752, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12261, "totalOutputTokens": 2183, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12197, "todayTokensCached": 12148, "todayTokensOut": 212, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12197, "totalTokensCached": 12148, "totalTokensOut": 212, "totalCost": 0, "medianOutputTokensPerSec": 25.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.3, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12197, "cachedInputTokens": 12148, "outputTokens": 212, "cost": 0, "durationMs": 116724, "outputTokensPerSec": 25.279916039313395, "at": "2026-10-30T01:01:48.124Z" }, "lastUpdated": "2026-10-30T01:01:48.124Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:01:51.760Z" } }, "source": "engine-log" }, "derived": { "durationMs": 120951, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 2183, "promptTokensPerSec": null, "peakRssMb": 17873, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12197, "output": 212 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 558, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-core-plan-and-estimate", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-02-14-249Z-r3ve", "success": true, "includedInCapability": true, "durationMs": 47861, "timing": { "startedAt": "2026-10-30T01:02:14.250Z", "finishedAt": "2026-10-30T01:03:02.111Z", "timeToFirstArtifactMs": 46248, "timeToLastArtifactWriteMs": 46248, "timeToFirstTokenMs": 12596, "firstTurnTtftMs": 10971, "timeToFirstToolCallMs": 31632 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 10381, "peakCpuPercent": 116.9, "firstAtMs": 1, "lastAtMs": 45165 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 68273, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5669, "totalOutputTokens": 947, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:03:01.010Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45691, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 947, "promptTokensPerSec": null, "peakRssMb": 10381, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1123, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-core-plan-and-estimate", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-30T01-03-23-751Z-xnx9", "success": true, "includedInCapability": true, "durationMs": 29821, "timing": { "startedAt": "2026-10-30T01:03:23.751Z", "finishedAt": "2026-10-30T01:03:53.572Z", "timeToFirstArtifactMs": 24376, "timeToLastArtifactWriteMs": 25614, "timeToFirstTokenMs": 10484, "firstTurnTtftMs": 8636, "timeToFirstToolCallMs": 15479 }, "metrics": { "process": { "sampleCount": 6, "peakRssMb": 14900, "peakCpuPercent": 99.7, "firstAtMs": 0, "lastAtMs": 25085 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 63535, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6186, "totalOutputTokens": 983, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6181, "todayTokensCached": 5649, "todayTokensOut": 34, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6181, "totalTokensCached": 5649, "totalTokensOut": 34, "totalCost": 0, "medianOutputTokensPerSec": 63.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 63.6, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6181, "cachedInputTokens": 5649, "outputTokens": 34, "cost": 0, "durationMs": 23766, "outputTokensPerSec": 63.55246737640009, "at": "2026-10-30T01:03:49.365Z" }, "lastUpdated": "2026-10-30T01:03:49.365Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:03:50.417Z" } }, "source": "engine-log" }, "derived": { "durationMs": 25616, "meanTokensPerSec": 71.22, "genTokensPerSec": 71.22, "genTokens": 983, "promptTokensPerSec": 987.89, "peakRssMb": 14900, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6181, "output": 34 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-plan-and-estimate", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-04-14-071Z-wx28", "success": true, "includedInCapability": true, "durationMs": 62784, "timing": { "startedAt": "2026-10-30T01:04:14.072Z", "finishedAt": "2026-10-30T01:05:16.856Z", "timeToFirstArtifactMs": 56549, "timeToLastArtifactWriteMs": 60704, "timeToFirstTokenMs": 15484, "firstTurnTtftMs": 13845, "timeToFirstToolCallMs": 16398 }, "metrics": { "process": { "sampleCount": 13, "peakRssMb": 17960, "peakCpuPercent": 119.4, "firstAtMs": 0, "lastAtMs": 60203 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 84075, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4855, "totalOutputTokens": 1107, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:05:15.869Z" } }, "source": "engine-log" }, "derived": { "durationMs": 60727, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 1107, "promptTokensPerSec": null, "peakRssMb": 17960, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 612, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-plan-and-estimate", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-qwen3-8-27b-q4-2026-10-30T01-05-38-600Z-42hr", "success": true, "includedInCapability": true, "durationMs": 73637, "timing": { "startedAt": "2026-10-30T01:05:38.601Z", "finishedAt": "2026-10-30T01:06:52.238Z", "timeToFirstArtifactMs": 67377, "timeToLastArtifactWriteMs": 67377, "timeToFirstTokenMs": 17325, "firstTurnTtftMs": 15518, "timeToFirstToolCallMs": 18521 }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 37169, "peakCpuPercent": 97.9, "firstAtMs": 0, "lastAtMs": 70229 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 88138, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4874, "totalOutputTokens": 1227, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:06:50.402Z" } }, "source": "engine-log" }, "derived": { "durationMs": 70753, "meanTokensPerSec": 25.41, "genTokensPerSec": 25.41, "genTokens": 1227, "promptTokensPerSec": 459.55, "peakRssMb": 37169, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-core-conflict-synthesis", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-30T01-07-13-637Z-5pe4", "success": true, "includedInCapability": true, "durationMs": 34996, "timing": { "startedAt": "2026-10-30T01:07:13.638Z", "finishedAt": "2026-10-30T01:07:48.634Z", "timeToFirstArtifactMs": 26947, "timeToLastArtifactWriteMs": 28838, "timeToFirstTokenMs": 14533, "firstTurnTtftMs": 12739, "timeToFirstToolCallMs": 20448 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 14997, "peakCpuPercent": 115.6, "firstAtMs": 0, "lastAtMs": 30096 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 63965, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 11093, "totalOutputTokens": 920, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11090, "todayTokensCached": 10655, "todayTokensOut": 68, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11090, "totalTokensCached": 10655, "totalTokensOut": 68, "totalCost": 0, "medianOutputTokensPerSec": 55.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 55.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11090, "cachedInputTokens": 10655, "outputTokens": 68, "cost": 0, "durationMs": 27044, "outputTokensPerSec": 55.18577426053122, "at": "2026-10-30T01:07:42.476Z" }, "lastUpdated": "2026-10-30T01:07:42.476Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:07:45.327Z" } }, "source": "engine-log" }, "derived": { "durationMs": 30638, "meanTokensPerSec": 72.43, "genTokensPerSec": 72.43, "genTokens": 920, "promptTokensPerSec": 1052.17, "peakRssMb": 14997, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11090, "output": 68 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-core-conflict-synthesis", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T01-08-06-141Z-slji", "success": true, "includedInCapability": true, "durationMs": 163432, "timing": { "startedAt": "2026-10-30T01:08:06.142Z", "finishedAt": "2026-10-30T01:10:49.574Z", "timeToFirstArtifactMs": 84898, "timeToLastArtifactWriteMs": 157603, "timeToFirstTokenMs": 26671, "firstTurnTtftMs": 25026, "timeToFirstToolCallMs": 28837 }, "metrics": { "process": { "sampleCount": 33, "peakRssMb": 17839, "peakCpuPercent": 115, "firstAtMs": 0, "lastAtMs": 160531 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 85563, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13470, "totalOutputTokens": 3634, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 13406, "todayTokensCached": 13325, "todayTokensOut": 504, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 13406, "totalTokensCached": 13325, "totalTokensOut": 504, "totalCost": 0, "medianOutputTokensPerSec": 26.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.1, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13406, "cachedInputTokens": 13325, "outputTokens": 504, "cost": 0, "durationMs": 155958, "outputTokensPerSec": 26.05540108656521, "at": "2026-10-30T01:10:43.745Z" }, "lastUpdated": "2026-10-30T01:10:43.745Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:10:48.289Z" } }, "source": "engine-log" }, "derived": { "durationMs": 161077, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 3634, "promptTokensPerSec": null, "peakRssMb": 17839, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 13406, "output": 504 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 615, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-core-conflict-synthesis", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-qwen3-8-27b-q4-2026-10-30T01-11-14-819Z-9tgj", "success": true, "includedInCapability": true, "durationMs": 224168, "timing": { "startedAt": "2026-10-30T01:11:14.819Z", "finishedAt": "2026-10-30T01:14:58.987Z", "timeToFirstArtifactMs": 142764, "timeToLastArtifactWriteMs": 220800, "timeToFirstTokenMs": 34978, "firstTurnTtftMs": 33141, "timeToFirstToolCallMs": 39762 }, "metrics": { "process": { "sampleCount": 45, "peakRssMb": 42841, "peakCpuPercent": 118.2, "firstAtMs": 0, "lastAtMs": 220746 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 45, "peakUsedMb": 91955, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 30098, "totalOutputTokens": 4270, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 20140, "todayTokensCached": 13146, "todayTokensOut": 1298, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 20140, "totalTokensCached": 13146, "totalTokensOut": 1298, "totalCost": 0, "medianOutputTokensPerSec": 31.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.6, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5896, "outputTokens": 859, "cost": 0, "durationMs": 53601, "outputTokensPerSec": 37.5199718416547, "at": "2026-10-30T01:14:55.619Z" }, "lastUpdated": "2026-10-30T01:14:55.619Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:14:57.155Z" } }, "source": "engine-log" }, "derived": { "durationMs": 221289, "meanTokensPerSec": 31.81, "genTokensPerSec": 31.81, "genTokens": 4270, "promptTokensPerSec": 396.66, "peakRssMb": 42841, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20140, "output": 1298 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-core-conflict-synthesis", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T01-15-21-615Z-vhak", "success": false, "includedInCapability": true, "durationMs": 273922, "timing": { "startedAt": "2026-10-30T01:15:21.616Z", "finishedAt": "2026-10-30T01:19:55.538Z", "timeToFirstArtifactMs": 48831, "timeToLastArtifactWriteMs": 268911, "timeToFirstTokenMs": 17853, "firstTurnTtftMs": 16218, "timeToFirstToolCallMs": 35570 }, "metrics": { "process": { "sampleCount": 55, "peakRssMb": 10685, "peakCpuPercent": 117.8, "firstAtMs": 0, "lastAtMs": 271066 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 55, "peakUsedMb": 68489, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 101863, "totalOutputTokens": 3384, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 55970, "todayTokensCached": 18742, "todayTokensOut": 1621, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 55970, "totalTokensCached": 18742, "totalTokensOut": 1621, "totalCost": 0, "medianOutputTokensPerSec": 35.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.2, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14260, "cachedInputTokens": 1180, "outputTokens": 247, "cost": 0, "durationMs": 41887, "outputTokensPerSec": 33.9328461822585, "at": "2026-10-30T01:19:50.531Z" }, "lastUpdated": "2026-10-30T01:19:50.531Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:19:54.291Z" } }, "source": "engine-log" }, "derived": { "durationMs": 271610, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 3384, "promptTokensPerSec": null, "peakRssMb": 10685, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 55970, "output": 1621 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1116.5, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-productivity-constrained-comms", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "constrained-comms", "repetition": 2 }, "trialId": "constrained-comms-mlx-qwen3-8-27b-q4-2026-10-30T01-20-16-922Z-1u4u", "success": true, "includedInCapability": true, "durationMs": 87816, "timing": { "startedAt": "2026-10-30T01:20:16.923Z", "finishedAt": "2026-10-30T01:21:44.739Z", "timeToFirstArtifactMs": 43378, "timeToLastArtifactWriteMs": 85084, "timeToFirstTokenMs": 29573, "firstTurnTtftMs": 27930, "timeToFirstToolCallMs": 31067 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 18152, "peakCpuPercent": 110.9, "firstAtMs": 0, "lastAtMs": 85307 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 86494, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 28885, "totalOutputTokens": 188, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 11663, "todayTokensCached": 1389, "todayTokensOut": 484, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 11663, "totalTokensCached": 1389, "totalTokensOut": 484, "totalCost": 0, "medianOutputTokensPerSec": 36.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.4, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4539, "cachedInputTokens": 1389, "outputTokens": 101, "cost": 0, "durationMs": 13648, "outputTokensPerSec": 36.44805224761674, "at": "2026-10-30T01:21:42.010Z" }, "lastUpdated": "2026-10-30T01:21:42.010Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:21:43.839Z" } }, "source": "engine-log" }, "derived": { "durationMs": 85852, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 188, "promptTokensPerSec": null, "peakRssMb": 18152, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11663, "output": 484 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 611, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-constrained-comms", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "constrained-comms", "repetition": 2 }, "trialId": "constrained-comms-qwen3-8-27b-q4-2026-10-30T01-22-06-466Z-dhay", "success": true, "includedInCapability": true, "durationMs": 84265, "timing": { "startedAt": "2026-10-30T01:22:06.467Z", "finishedAt": "2026-10-30T01:23:30.732Z", "timeToFirstArtifactMs": 48151, "timeToLastArtifactWriteMs": 78623, "timeToFirstTokenMs": 34028, "firstTurnTtftMs": 32201, "timeToFirstToolCallMs": 35191 }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 39906, "peakCpuPercent": 120.9, "firstAtMs": 0, "lastAtMs": 80294 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 89560, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 21390, "totalOutputTokens": 663, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 15484, "todayTokensCached": 11656, "todayTokensOut": 549, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 15484, "totalTokensCached": 11656, "totalTokensOut": 549, "totalCost": 0, "medianOutputTokensPerSec": 37.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.7, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 6570, "cachedInputTokens": 6343, "outputTokens": 151, "cost": 0, "durationMs": 7206, "outputTokensPerSec": 36.12502727439559, "at": "2026-10-30T01:23:25.093Z" }, "lastUpdated": "2026-10-30T01:23:25.093Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:23:28.357Z" } }, "source": "engine-log" }, "derived": { "durationMs": 80840, "meanTokensPerSec": 40.53, "genTokensPerSec": 40.53, "genTokens": 663, "promptTokensPerSec": 456.02, "peakRssMb": 39906, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15484, "output": 549 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-constrained-comms", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "constrained-comms", "repetition": 2 }, "trialId": "constrained-comms-mlx-gemma4-12b-q4-2026-10-30T01-23-52-517Z-d5qq", "success": true, "includedInCapability": true, "durationMs": 52724, "timing": { "startedAt": "2026-10-30T01:23:52.517Z", "finishedAt": "2026-10-30T01:24:45.241Z", "timeToFirstArtifactMs": 36002, "timeToLastArtifactWriteMs": 47655, "timeToFirstTokenMs": 16451, "firstTurnTtftMs": 14792, "timeToFirstToolCallMs": 26108 }, "metrics": { "process": { "sampleCount": 11, "peakRssMb": 10604, "peakCpuPercent": 102, "firstAtMs": 0, "lastAtMs": 50195 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 11, "peakUsedMb": 68449, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 21257, "totalOutputTokens": 408, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 11336, "todayTokensCached": 0, "todayTokensOut": 434, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 11336, "totalTokensCached": 0, "totalTokensOut": 434, "totalCost": 0, "medianOutputTokensPerSec": 40.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5785, "cachedInputTokens": 0, "outputTokens": 243, "cost": 0, "durationMs": 10854, "outputTokensPerSec": 40.569908016472965, "at": "2026-10-30T01:24:40.172Z" }, "lastUpdated": "2026-10-30T01:24:40.172Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:24:44.322Z" } }, "source": "engine-log" }, "derived": { "durationMs": 50737, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 408, "promptTokensPerSec": null, "peakRssMb": 10604, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11336, "output": 434 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1190.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-constrained-comms", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "constrained-comms", "repetition": 2 }, "trialId": "constrained-comms-gemma4-12b-q4-2026-10-30T01-25-21-144Z-uzms", "success": true, "includedInCapability": true, "durationMs": 27860, "timing": { "startedAt": "2026-10-30T01:25:21.145Z", "finishedAt": "2026-10-30T01:25:49.005Z", "timeToFirstArtifactMs": 25619, "timeToLastArtifactWriteMs": 25621, "timeToFirstTokenMs": 14434, "firstTurnTtftMs": 12685, "timeToFirstToolCallMs": 17889 }, "metrics": { "process": { "sampleCount": 6, "peakRssMb": 15153, "peakCpuPercent": 95.5, "firstAtMs": 0, "lastAtMs": 25082 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 63524, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15528, "totalOutputTokens": 533, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5605, "todayTokensCached": 0, "todayTokensOut": 215, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5605, "totalTokensCached": 0, "totalTokensOut": 215, "totalCost": 0, "medianOutputTokensPerSec": 85, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 85, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5605, "outputTokens": 215, "cost": 0, "durationMs": 23872, "outputTokensPerSec": 84.95340266166157, "at": "2026-10-30T01:25:46.766Z" }, "lastUpdated": "2026-10-30T01:25:46.766Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:25:47.803Z" } }, "source": "engine-log" }, "derived": { "durationMs": 25616, "meanTokensPerSec": 88.94, "genTokensPerSec": 88.94, "genTokens": 533, "promptTokensPerSec": 1102.3, "peakRssMb": 15153, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5605, "output": 215 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-productivity-records-intake", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "records-intake", "repetition": 2 }, "trialId": "records-intake-qwen3-8-27b-q4-2026-10-30T01-26-10-215Z-4z9z", "success": true, "includedInCapability": true, "durationMs": 159164, "timing": { "startedAt": "2026-10-30T01:26:10.216Z", "finishedAt": "2026-10-30T01:28:49.380Z", "timeToFirstArtifactMs": 152480, "timeToLastArtifactWriteMs": 152495, "timeToFirstTokenMs": 17087, "firstTurnTtftMs": 15305, "timeToFirstToolCallMs": 18646 }, "metrics": { "process": { "sampleCount": 32, "peakRssMb": 37257, "peakCpuPercent": 103.8, "firstAtMs": 0, "lastAtMs": 155515 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 88030, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5677, "totalOutputTokens": 5401, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:28:47.305Z" } }, "source": "engine-log" }, "derived": { "durationMs": 156037, "meanTokensPerSec": 40.74, "genTokensPerSec": 40.74, "genTokens": 5401, "promptTokensPerSec": 495.32, "peakRssMb": 37257, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-records-intake", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "records-intake", "repetition": 2 }, "trialId": "records-intake-mlx-gemma4-12b-q4-2026-10-30T01-29-09-739Z-druf", "success": true, "includedInCapability": true, "durationMs": 233423, "timing": { "startedAt": "2026-10-30T01:29:09.740Z", "finishedAt": "2026-10-30T01:33:03.163Z", "timeToFirstArtifactMs": 72205, "timeToLastArtifactWriteMs": 229093, "timeToFirstTokenMs": 12692, "firstTurnTtftMs": 11060, "timeToFirstToolCallMs": 36825 }, "metrics": { "process": { "sampleCount": 47, "peakRssMb": 10371, "peakCpuPercent": 101.6, "firstAtMs": 1, "lastAtMs": 230913 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 47, "peakUsedMb": 68752, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18758, "totalOutputTokens": 6206, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 8743, "todayTokensCached": 7521, "todayTokensOut": 173, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 8743, "totalTokensCached": 7521, "totalTokensOut": 173, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8743, "cachedInputTokens": 7521, "outputTokens": 173, "cost": 0, "durationMs": 113706, "outputTokensPerSec": 35.774792171981844, "at": "2026-10-30T01:31:05.078Z" }, "lastUpdated": "2026-10-30T01:31:05.078Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:33:02.245Z" } }, "source": "engine-log" }, "derived": { "durationMs": 231438, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 6206, "promptTokensPerSec": null, "peakRssMb": 10371, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 8743, "output": 173 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1087, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-records-intake", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "records-intake", "repetition": 2 }, "trialId": "records-intake-gemma4-12b-q4-2026-10-30T01-33-28-671Z-swq8", "success": true, "includedInCapability": true, "durationMs": 129666, "timing": { "startedAt": "2026-10-30T01:33:28.672Z", "finishedAt": "2026-10-30T01:35:38.338Z", "timeToFirstArtifactMs": 43027, "timeToLastArtifactWriteMs": 124495, "timeToFirstTokenMs": 10620, "firstTurnTtftMs": 8865, "timeToFirstToolCallMs": 20040 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 20298, "peakCpuPercent": 118.9, "firstAtMs": 0, "lastAtMs": 125487 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 68464, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17301, "totalOutputTokens": 6821, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 17278, "todayTokensCached": 15000, "todayTokensOut": 133, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 17278, "totalTokensCached": 15000, "totalTokensOut": 133, "totalCost": 0, "medianOutputTokensPerSec": 61.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 61.7, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7584, "cachedInputTokens": 7018, "outputTokens": 46, "cost": 0, "durationMs": 37617, "outputTokensPerSec": 53.98479788091674, "at": "2026-10-30T01:35:33.167Z" }, "lastUpdated": "2026-10-30T01:35:33.167Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:35:35.646Z" } }, "source": "engine-log" }, "derived": { "durationMs": 126030, "meanTokensPerSec": 86.27, "genTokensPerSec": 86.27, "genTokens": 6821, "promptTokensPerSec": 928.25, "peakRssMb": 20298, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 17278, "output": 133 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-records-intake", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "records-intake", "repetition": 2 }, "trialId": "records-intake-mlx-qwen3-8-27b-q4-2026-10-30T01-36-02-730Z-685a", "success": true, "includedInCapability": true, "durationMs": 82793, "timing": { "startedAt": "2026-10-30T01:36:02.731Z", "finishedAt": "2026-10-30T01:37:25.524Z", "timeToFirstArtifactMs": 80177, "timeToLastArtifactWriteMs": 80177, "timeToFirstTokenMs": 15031, "firstTurnTtftMs": 13401, "timeToFirstToolCallMs": 16398 }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 17982, "peakCpuPercent": 116.3, "firstAtMs": 1, "lastAtMs": 80265 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 84203, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5562, "totalOutputTokens": 2277, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:37:24.583Z" } }, "source": "engine-log" }, "derived": { "durationMs": 80787, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 2277, "promptTokensPerSec": null, "peakRssMb": 17982, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 615, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-productivity-plan-and-estimate", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-37-46-515Z-5dua", "success": true, "includedInCapability": true, "durationMs": 47625, "timing": { "startedAt": "2026-10-30T01:37:46.516Z", "finishedAt": "2026-10-30T01:38:34.141Z", "timeToFirstArtifactMs": 43609, "timeToLastArtifactWriteMs": 45334, "timeToFirstTokenMs": 12089, "firstTurnTtftMs": 10469, "timeToFirstToolCallMs": 30846 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 10391, "peakCpuPercent": 121, "firstAtMs": 0, "lastAtMs": 45158 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 68131, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6220, "totalOutputTokens": 1041, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6172, "todayTokensCached": 5674, "todayTokensOut": 44, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6172, "totalTokensCached": 5674, "totalTokensOut": 44, "totalCost": 0, "medianOutputTokensPerSec": 43.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 43.5, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6172, "cachedInputTokens": 5674, "outputTokens": 44, "cost": 0, "durationMs": 43713, "outputTokensPerSec": 43.52303990050553, "at": "2026-10-30T01:38:31.849Z" }, "lastUpdated": "2026-10-30T01:38:31.849Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:38:33.267Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45692, "meanTokensPerSec": 42, "genTokensPerSec": 42, "genTokens": 1041, "promptTokensPerSec": null, "peakRssMb": 10391, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 6172, "output": 44 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1261, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-plan-and-estimate", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-30T01-38-54-826Z-qm89", "success": true, "includedInCapability": true, "durationMs": 29979, "timing": { "startedAt": "2026-10-30T01:38:54.827Z", "finishedAt": "2026-10-30T01:39:24.806Z", "timeToFirstArtifactMs": 22885, "timeToLastArtifactWriteMs": 24257, "timeToFirstTokenMs": 10098, "firstTurnTtftMs": 8237, "timeToFirstToolCallMs": 14246 }, "metrics": { "process": { "sampleCount": 6, "peakRssMb": 14884, "peakCpuPercent": 111.3, "firstAtMs": 0, "lastAtMs": 25088 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 63632, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6205, "totalOutputTokens": 969, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6200, "todayTokensCached": 5649, "todayTokensOut": 45, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6200, "totalTokensCached": 5649, "totalTokensOut": 45, "totalCost": 0, "medianOutputTokensPerSec": 70.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.1, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6200, "cachedInputTokens": 5649, "outputTokens": 45, "cost": 0, "durationMs": 22396, "outputTokensPerSec": 70.12198037222022, "at": "2026-10-30T01:39:19.084Z" }, "lastUpdated": "2026-10-30T01:39:19.084Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:39:21.492Z" } }, "source": "engine-log" }, "derived": { "durationMs": 25613, "meanTokensPerSec": 75.54, "genTokensPerSec": 75.54, "genTokens": 969, "promptTokensPerSec": 1059.03, "peakRssMb": 14884, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6200, "output": 45 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-plan-and-estimate", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-39-45-725Z-zfdd", "success": true, "includedInCapability": true, "durationMs": 72805, "timing": { "startedAt": "2026-10-30T01:39:45.725Z", "finishedAt": "2026-10-30T01:40:58.530Z", "timeToFirstArtifactMs": 67144, "timeToLastArtifactWriteMs": 67144, "timeToFirstTokenMs": 15146, "firstTurnTtftMs": 13505, "timeToFirstToolCallMs": 16446 }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 17977, "peakCpuPercent": 118.3, "firstAtMs": 1, "lastAtMs": 70238 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 84239, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4741, "totalOutputTokens": 1487, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:40:57.552Z" } }, "source": "engine-log" }, "derived": { "durationMs": 70762, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 1487, "promptTokensPerSec": null, "peakRssMb": 17977, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 642, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-plan-and-estimate", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 2 }, "trialId": "plan-and-estimate-qwen3-8-27b-q4-2026-10-30T01-41-43-630Z-8gim", "success": true, "includedInCapability": true, "durationMs": 93443, "timing": { "startedAt": "2026-10-30T01:41:43.630Z", "finishedAt": "2026-10-30T01:43:17.073Z", "timeToFirstArtifactMs": 90970, "timeToLastArtifactWriteMs": 90970, "timeToFirstTokenMs": 17151, "firstTurnTtftMs": 15424, "timeToFirstToolCallMs": 18886 }, "metrics": { "process": { "sampleCount": 19, "peakRssMb": 37027, "peakCpuPercent": 120.4, "firstAtMs": 0, "lastAtMs": 90297 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 19, "peakUsedMb": 87662, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4750, "totalOutputTokens": 1887, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:43:15.490Z" } }, "source": "engine-log" }, "derived": { "durationMs": 90820, "meanTokensPerSec": 25.93, "genTokensPerSec": 25.93, "genTokens": 1887, "promptTokensPerSec": 484.23, "peakRssMb": 37027, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-productivity-meeting-followup", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "meeting-followup", "repetition": 2 }, "trialId": "meeting-followup-gemma4-12b-q4-2026-10-30T01-43-39-254Z-f0ck", "success": true, "includedInCapability": true, "durationMs": 176746, "timing": { "startedAt": "2026-10-30T01:43:39.254Z", "finishedAt": "2026-10-30T01:46:36.000Z", "timeToFirstArtifactMs": 24762, "timeToLastArtifactWriteMs": 167451, "timeToFirstTokenMs": 12331, "firstTurnTtftMs": 10547, "timeToFirstToolCallMs": 17081 }, "metrics": { "process": { "sampleCount": 35, "peakRssMb": 19894, "peakCpuPercent": 119.9, "firstAtMs": 0, "lastAtMs": 170889 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 35, "peakUsedMb": 69019, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 23393, "totalOutputTokens": 4521, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 33022, "todayTokensCached": 27206, "todayTokensOut": 2427, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 33022, "totalTokensCached": 27206, "totalTokensOut": 2427, "totalCost": 0, "medianOutputTokensPerSec": 98.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 98.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10185, "cachedInputTokens": 9639, "outputTokens": 2080, "cost": 0, "durationMs": 20551, "outputTokensPerSec": 104.61820168507265, "at": "2026-10-30T01:46:26.705Z" }, "lastUpdated": "2026-10-30T01:46:26.705Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:46:31.724Z" } }, "source": "engine-log" }, "derived": { "durationMs": 171419, "meanTokensPerSec": 100.34, "genTokensPerSec": 100.34, "genTokens": 4521, "promptTokensPerSec": 1161.23, "peakRssMb": 19894, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 33022, "output": 2427 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-meeting-followup", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "meeting-followup", "repetition": 2 }, "trialId": "meeting-followup-mlx-qwen3-8-27b-q4-2026-10-30T01-46-57-377Z-izhl", "success": true, "includedInCapability": true, "durationMs": 138314, "timing": { "startedAt": "2026-10-30T01:46:57.378Z", "finishedAt": "2026-10-30T01:49:15.692Z", "timeToFirstArtifactMs": 41244, "timeToLastArtifactWriteMs": 134003, "timeToFirstTokenMs": 17188, "firstTurnTtftMs": 15558, "timeToFirstToolCallMs": 26351 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 18164, "peakCpuPercent": 129.3, "firstAtMs": 0, "lastAtMs": 135723 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 85763, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12678, "totalOutputTokens": 693, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 2883, "todayTokensCached": 0, "todayTokensOut": 502, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 2883, "totalTokensCached": 0, "totalTokensOut": 502, "totalCost": 0, "medianOutputTokensPerSec": 47, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 47, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 2883, "cachedInputTokens": 0, "outputTokens": 502, "cost": 0, "durationMs": 39616, "outputTokensPerSec": 46.99739307543802, "at": "2026-10-30T01:47:38.624Z" }, "lastUpdated": "2026-10-30T01:47:38.624Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:49:14.693Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136243, "meanTokensPerSec": 47, "genTokensPerSec": 47, "genTokens": 693, "promptTokensPerSec": null, "peakRssMb": 18164, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 2883, "output": 502 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 722.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-meeting-followup", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "meeting-followup", "repetition": 2 }, "trialId": "meeting-followup-qwen3-8-27b-q4-2026-10-30T01-49-37-138Z-qntb", "success": true, "includedInCapability": true, "durationMs": 139442, "timing": { "startedAt": "2026-10-30T01:49:37.139Z", "finishedAt": "2026-10-30T01:51:56.581Z", "timeToFirstArtifactMs": 38066, "timeToLastArtifactWriteMs": 135518, "timeToFirstTokenMs": 20120, "firstTurnTtftMs": 18311, "timeToFirstToolCallMs": 22127 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 38581, "peakCpuPercent": 118.2, "firstAtMs": 0, "lastAtMs": 135733 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 89696, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12222, "totalOutputTokens": 982, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 2822, "todayTokensCached": 0, "todayTokensOut": 514, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 2822, "totalTokensCached": 0, "totalTokensOut": 514, "totalCost": 0, "medianOutputTokensPerSec": 48.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 48.4, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 2822, "outputTokens": 514, "cost": 0, "durationMs": 36259, "outputTokensPerSec": 48.43888462150393, "at": "2026-10-30T01:50:15.207Z" }, "lastUpdated": "2026-10-30T01:50:15.207Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:51:54.446Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136256, "meanTokensPerSec": 54.41, "genTokensPerSec": 54.41, "genTokens": 982, "promptTokensPerSec": 608.8, "peakRssMb": 38581, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 2822, "output": 514 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-meeting-followup", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "meeting-followup", "repetition": 2 }, "trialId": "meeting-followup-mlx-gemma4-12b-q4-2026-10-30T01-52-17-930Z-hf22", "success": true, "includedInCapability": true, "durationMs": 208602, "timing": { "startedAt": "2026-10-30T01:52:17.930Z", "finishedAt": "2026-10-30T01:55:46.532Z", "timeToFirstArtifactMs": 28217, "timeToLastArtifactWriteMs": 203068, "timeToFirstTokenMs": 11549, "firstTurnTtftMs": 9913, "timeToFirstToolCallMs": 20968 }, "metrics": { "process": { "sampleCount": 42, "peakRssMb": 10543, "peakCpuPercent": 100.5, "firstAtMs": 0, "lastAtMs": 205958 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 42, "peakUsedMb": 68302, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 27031, "totalOutputTokens": 3427, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 11435, "todayTokensCached": 0, "todayTokensOut": 275, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 11435, "totalTokensCached": 0, "totalTokensOut": 275, "totalCost": 0, "medianOutputTokensPerSec": 50.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 50.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6935, "cachedInputTokens": 0, "outputTokens": 70, "cost": 0, "durationMs": 6034, "outputTokensPerSec": 49.93486756078902, "at": "2026-10-30T01:55:41.003Z" }, "lastUpdated": "2026-10-30T01:55:41.003Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T01:55:45.497Z" } }, "source": "engine-log" }, "derived": { "durationMs": 206502, "meanTokensPerSec": 49, "genTokensPerSec": 49, "genTokens": 3427, "promptTokensPerSec": null, "peakRssMb": 10543, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11435, "output": 275 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1921, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-productivity-craftbook-spreadsheet-model", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 2 }, "trialId": "craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-30T01-56-09-353Z-t72x", "success": true, "includedInCapability": true, "durationMs": 2371250, "timing": { "startedAt": "2026-10-30T01:56:09.354Z", "finishedAt": "2026-10-30T02:35:40.604Z", "timeToFirstArtifactMs": 101137, "timeToLastArtifactWriteMs": 2356348, "timeToFirstTokenMs": 28173, "firstTurnTtftMs": 21669, "timeToFirstToolCallMs": 31080 }, "metrics": { "process": { "sampleCount": 471, "peakRssMb": 18832, "peakCpuPercent": 105.5, "firstAtMs": 0, "lastAtMs": 2357874 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 471, "peakUsedMb": 84022, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 89372, "totalOutputTokens": 48025, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 62650, "todayTokensCached": 58675, "todayTokensOut": 3713, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 62650, "totalTokensCached": 58675, "totalTokensOut": 3713, "totalCost": 0, "medianOutputTokensPerSec": 23.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.6, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 23013, "cachedInputTokens": 20357, "outputTokens": 641, "cost": 0, "durationMs": 178992, "outputTokensPerSec": 23.83499713031395, "at": "2026-10-30T02:35:25.702Z" }, "lastUpdated": "2026-10-30T02:35:25.702Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T02:35:28.821Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2358400, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 48025, "promptTokensPerSec": null, "peakRssMb": 18832, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 62650, "output": 3713 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 530, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-craftbook-spreadsheet-model", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 2 }, "trialId": "craftbook-spreadsheet-model-qwen3-8-27b-q4-2026-10-30T02-36-03-416Z-0lvx", "success": true, "includedInCapability": true, "durationMs": 539154, "timing": { "startedAt": "2026-10-30T02:36:03.417Z", "finishedAt": "2026-10-30T02:45:02.571Z", "timeToFirstArtifactMs": 122265, "timeToLastArtifactWriteMs": 528463, "timeToFirstTokenMs": 31920, "firstTurnTtftMs": 25239, "timeToFirstToolCallMs": 40176 }, "metrics": { "process": { "sampleCount": 107, "peakRssMb": 43366, "peakCpuPercent": 106.8, "firstAtMs": 0, "lastAtMs": 531737 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 107, "peakUsedMb": 98338, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44974, "totalOutputTokens": 11289, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 44938, "todayTokensCached": 36128, "todayTokensOut": 3424, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 44938, "totalTokensCached": 36128, "totalTokensOut": 3424, "totalCost": 0, "medianOutputTokensPerSec": 27.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27.6, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 17772, "cachedInputTokens": 16569, "outputTokens": 1006, "cost": 0, "durationMs": 151840, "outputTokensPerSec": 27.594662253277978, "at": "2026-10-30T02:44:51.880Z" }, "lastUpdated": "2026-10-30T02:44:51.880Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T02:44:56.732Z" } }, "source": "engine-log" }, "derived": { "durationMs": 532262, "meanTokensPerSec": 29.53, "genTokensPerSec": 29.53, "genTokens": 11289, "promptTokensPerSec": 383.75, "peakRssMb": 43366, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44938, "output": 3424 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-craftbook-spreadsheet-model", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 2 }, "trialId": "craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-30T02-45-24-448Z-z5fs", "success": true, "includedInCapability": true, "durationMs": 378772, "timing": { "startedAt": "2026-10-30T02:45:24.449Z", "finishedAt": "2026-10-30T02:51:43.221Z", "timeToFirstArtifactMs": 45298, "timeToLastArtifactWriteMs": 335463, "timeToFirstTokenMs": 20304, "firstTurnTtftMs": 13797, "timeToFirstToolCallMs": 45297 }, "metrics": { "process": { "sampleCount": 75, "peakRssMb": 12263, "peakCpuPercent": 129, "firstAtMs": 0, "lastAtMs": 371492 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 75, "peakUsedMb": 69151, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 77715, "totalOutputTokens": 7090, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 73747, "todayTokensCached": 59359, "todayTokensOut": 1763, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 73747, "totalTokensCached": 59359, "totalTokensOut": 1763, "totalCost": 0, "medianOutputTokensPerSec": 36.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.4, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9078, "cachedInputTokens": 8881, "outputTokens": 35, "cost": 0, "durationMs": 22646, "outputTokensPerSec": 34.23079401148583, "at": "2026-10-30T02:51:23.639Z" }, "lastUpdated": "2026-10-30T02:51:23.639Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T02:51:37.532Z" } }, "source": "engine-log" }, "derived": { "durationMs": 372018, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 7090, "promptTokensPerSec": null, "peakRssMb": 12263, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 73747, "output": 1763 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1196, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-craftbook-spreadsheet-model", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 2 }, "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-30T02-52-05-736Z-qeqk", "success": true, "includedInCapability": true, "durationMs": 205934, "timing": { "startedAt": "2026-10-30T02:52:05.737Z", "finishedAt": "2026-10-30T02:55:31.671Z", "timeToFirstArtifactMs": 33572, "timeToLastArtifactWriteMs": 164771, "timeToFirstTokenMs": 19237, "firstTurnTtftMs": 12493, "timeToFirstToolCallMs": 33571 }, "metrics": { "process": { "sampleCount": 41, "peakRssMb": 24589, "peakCpuPercent": 114.9, "firstAtMs": 0, "lastAtMs": 200678 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 41, "peakUsedMb": 78670, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 63342, "totalOutputTokens": 6876, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44441, "todayTokensCached": 34873, "todayTokensOut": 1513, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44441, "totalTokensCached": 34873, "totalTokensOut": 1513, "totalCost": 0, "medianOutputTokensPerSec": 70.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.2, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8481, "cachedInputTokens": 8311, "outputTokens": 35, "cost": 0, "durationMs": 19790, "outputTokensPerSec": 87.94892754043063, "at": "2026-10-30T02:55:12.134Z" }, "lastUpdated": "2026-10-30T02:55:12.134Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T02:55:27.990Z" } }, "source": "engine-log" }, "derived": { "durationMs": 201203, "meanTokensPerSec": 71.07, "genTokensPerSec": 71.07, "genTokens": 6876, "promptTokensPerSec": 979.6, "peakRssMb": 24589, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44441, "output": 1513 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-productivity-conflict-synthesis", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-qwen3-8-27b-q4-2026-10-30T02-55-53-337Z-j2db", "success": true, "includedInCapability": true, "durationMs": 230841, "timing": { "startedAt": "2026-10-30T02:55:53.338Z", "finishedAt": "2026-10-30T02:59:44.179Z", "timeToFirstArtifactMs": 139253, "timeToLastArtifactWriteMs": 225839, "timeToFirstTokenMs": 36288, "firstTurnTtftMs": 34529, "timeToFirstToolCallMs": 42077 }, "metrics": { "process": { "sampleCount": 46, "peakRssMb": 39778, "peakCpuPercent": 112, "firstAtMs": 0, "lastAtMs": 225771 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 90480, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18977, "totalOutputTokens": 4336, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 14068, "todayTokensCached": 13903, "todayTokensOut": 367, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 14068, "totalTokensCached": 13903, "totalTokensOut": 367, "totalCost": 0, "medianOutputTokensPerSec": 19.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 19.2, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14068, "cachedInputTokens": 13903, "outputTokens": 367, "cost": 0, "durationMs": 224080, "outputTokensPerSec": 19.23722105832363, "at": "2026-10-30T02:59:39.177Z" }, "lastUpdated": "2026-10-30T02:59:39.177Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T02:59:40.698Z" } }, "source": "engine-log" }, "derived": { "durationMs": 226314, "meanTokensPerSec": 26.17, "genTokensPerSec": 26.17, "genTokens": 4336, "promptTokensPerSec": 369.55, "peakRssMb": 39778, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14068, "output": 367 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-conflict-synthesis", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T03-00-07-646Z-dpj6", "success": true, "includedInCapability": true, "durationMs": 123116, "timing": { "startedAt": "2026-10-30T03:00:07.647Z", "finishedAt": "2026-10-30T03:02:10.763Z", "timeToFirstArtifactMs": 45025, "timeToLastArtifactWriteMs": 119357, "timeToFirstTokenMs": 17952, "firstTurnTtftMs": 16277, "timeToFirstToolCallMs": 31409 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 10724, "peakCpuPercent": 120.2, "firstAtMs": 0, "lastAtMs": 120463 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 68546, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 39581, "totalOutputTokens": 1432, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 26868, "todayTokensCached": 18596, "todayTokensOut": 819, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 26868, "totalTokensCached": 18596, "totalTokensOut": 819, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8700, "cachedInputTokens": 1179, "outputTokens": 523, "cost": 0, "durationMs": 32847, "outputTokensPerSec": 35.80575245499376, "at": "2026-10-30T03:02:07.009Z" }, "lastUpdated": "2026-10-30T03:02:07.009Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:02:09.717Z" } }, "source": "engine-log" }, "derived": { "durationMs": 121010, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 1432, "promptTokensPerSec": null, "peakRssMb": 10724, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 26868, "output": 819 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1232, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-conflict-synthesis", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-30T03-02-36-742Z-tokz", "success": true, "includedInCapability": true, "durationMs": 199472, "timing": { "startedAt": "2026-10-30T03:02:36.743Z", "finishedAt": "2026-10-30T03:05:56.215Z", "timeToFirstArtifactMs": 26741, "timeToLastArtifactWriteMs": 192108, "timeToFirstTokenMs": 15500, "firstTurnTtftMs": 13533, "timeToFirstToolCallMs": 20255 }, "metrics": { "process": { "sampleCount": 40, "peakRssMb": 23673, "peakCpuPercent": 101.7, "firstAtMs": 0, "lastAtMs": 195677 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 71864, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 142902, "totalOutputTokens": 2459, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 35756, "todayTokensCached": 17237, "todayTokensOut": 1397, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 35756, "totalTokensCached": 17237, "totalTokensOut": 1397, "totalCost": 0, "medianOutputTokensPerSec": 76.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 76.8, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9838, "outputTokens": 465, "cost": 0, "durationMs": 65385, "outputTokensPerSec": 81.4821032638148, "at": "2026-10-30T03:05:48.851Z" }, "lastUpdated": "2026-10-30T03:05:48.851Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:05:54.113Z" } }, "source": "engine-log" }, "derived": { "durationMs": 196217, "meanTokensPerSec": 79.04, "genTokensPerSec": 79.04, "genTokens": 2459, "promptTokensPerSec": 1006.12, "peakRssMb": 23673, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35756, "output": 1397 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-conflict-synthesis", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 2 }, "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T03-06-17-524Z-khzv", "success": true, "includedInCapability": true, "durationMs": 359082, "timing": { "startedAt": "2026-10-30T03:06:17.524Z", "finishedAt": "2026-10-30T03:12:16.606Z", "timeToFirstArtifactMs": 169558, "timeToLastArtifactWriteMs": 352943, "timeToFirstTokenMs": 30211, "firstTurnTtftMs": 28560, "timeToFirstToolCallMs": 35167 }, "metrics": { "process": { "sampleCount": 72, "peakRssMb": 18014, "peakCpuPercent": 125.5, "firstAtMs": 0, "lastAtMs": 356224 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 72, "peakUsedMb": 85343, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 38799, "totalOutputTokens": 5994, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22862, "todayTokensCached": 18133, "todayTokensOut": 1702, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22862, "totalTokensCached": 18133, "totalTokensOut": 1702, "totalCost": 0, "medianOutputTokensPerSec": 28.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.7, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5914, "cachedInputTokens": 1474, "outputTokens": 808, "cost": 0, "durationMs": 70842, "outputTokensPerSec": 29.97441431115522, "at": "2026-10-30T03:12:10.466Z" }, "lastUpdated": "2026-10-30T03:12:10.466Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:12:15.359Z" } }, "source": "engine-log" }, "derived": { "durationMs": 356766, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 5994, "promptTokensPerSec": null, "peakRssMb": 18014, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 22862, "output": 1702 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 574, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-productivity-docblocks-theme-roundtrip", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 2 }, "trialId": "docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-30T03-12-38-000Z-qua9", "success": false, "includedInCapability": true, "durationMs": 529666, "timing": { "startedAt": "2026-10-30T03:12:38.000Z", "finishedAt": "2026-10-30T03:21:27.666Z", "timeToFirstArtifactMs": 109535, "timeToLastArtifactWriteMs": 527978, "timeToFirstTokenMs": 13280, "firstTurnTtftMs": 11638, "timeToFirstToolCallMs": 106008 }, "metrics": { "process": { "sampleCount": 106, "peakRssMb": 10469, "peakCpuPercent": 102.5, "firstAtMs": 0, "lastAtMs": 527187 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 106, "peakUsedMb": 68492, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 67538, "totalOutputTokens": 12036, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 58388, "todayTokensCached": 16349, "todayTokensOut": 1734, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 58388, "totalTokensCached": 16349, "totalTokensOut": 1734, "totalCost": 0, "medianOutputTokensPerSec": 33.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 33.3, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10965, "cachedInputTokens": 525, "outputTokens": 132, "cost": 0, "durationMs": 14724, "outputTokensPerSec": 33.52144614531936, "at": "2026-10-30T03:21:25.978Z" }, "lastUpdated": "2026-10-30T03:21:25.978Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:21:26.794Z" } }, "source": "engine-log" }, "derived": { "durationMs": 527731, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 12036, "promptTokensPerSec": null, "peakRssMb": 10469, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 58388, "output": 1734 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1045, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-docblocks-theme-roundtrip", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 2 }, "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-30T03-21-52-915Z-1y4o", "success": false, "includedInCapability": true, "durationMs": 866430, "timing": { "startedAt": "2026-10-30T03:21:52.915Z", "finishedAt": "2026-10-30T03:36:19.345Z", "timeToFirstArtifactMs": 98298, "timeToLastArtifactWriteMs": 519140, "timeToFirstTokenMs": 11434, "firstTurnTtftMs": 9567, "timeToFirstToolCallMs": 31386 }, "metrics": { "process": { "sampleCount": 173, "peakRssMb": 25239, "peakCpuPercent": 107.9, "firstAtMs": 0, "lastAtMs": 864723 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 173, "peakUsedMb": 73621, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 113001, "totalOutputTokens": 12335, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 104376, "todayTokensCached": 62484, "todayTokensOut": 5043, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 104376, "totalTokensCached": 62484, "totalTokensOut": 5043, "totalCost": 0, "medianOutputTokensPerSec": 87.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 87.7, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13341, "outputTokens": 166, "cost": 0, "durationMs": 35461, "outputTokensPerSec": 102.95782223614407, "at": "2026-10-30T03:31:12.161Z" }, "lastUpdated": "2026-10-30T03:31:12.161Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:36:19.236Z" } }, "source": "engine-log" }, "derived": { "durationMs": 865269, "meanTokensPerSec": 85.88, "genTokensPerSec": 85.88, "genTokens": 12335, "promptTokensPerSec": 1178.13, "peakRssMb": 25239, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 104376, "output": 5043 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-docblocks-theme-roundtrip", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 2 }, "trialId": "docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-30T03-36-40-892Z-0z2j", "success": false, "includedInCapability": true, "durationMs": 298840, "timing": { "startedAt": "2026-10-30T03:36:40.893Z", "finishedAt": "2026-10-30T03:41:39.733Z", "timeToFirstArtifactMs": 39727, "timeToLastArtifactWriteMs": 293801, "timeToFirstTokenMs": 13675, "firstTurnTtftMs": 12155, "timeToFirstToolCallMs": 14625 }, "metrics": { "process": { "sampleCount": 60, "peakRssMb": 17975, "peakCpuPercent": 119.9, "firstAtMs": 0, "lastAtMs": 296092 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 60, "peakUsedMb": 85483, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 98634, "totalOutputTokens": 2770, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 67564, "todayTokensCached": 18471, "todayTokensOut": 2947, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 67564, "totalTokensCached": 18471, "totalTokensOut": 2947, "totalCost": 0, "medianOutputTokensPerSec": 40.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 40.1, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11909, "cachedInputTokens": 1220, "outputTokens": 144, "cost": 0, "durationMs": 51890, "outputTokensPerSec": 22.861054894123786, "at": "2026-10-30T03:41:34.696Z" }, "lastUpdated": "2026-10-30T03:41:34.696Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:41:38.487Z" } }, "source": "engine-log" }, "derived": { "durationMs": 296637, "meanTokensPerSec": 42, "genTokensPerSec": 42, "genTokens": 2770, "promptTokensPerSec": null, "peakRssMb": 17975, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 67564, "output": 2947 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 634, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-docblocks-theme-roundtrip", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 2 }, "trialId": "docblocks-theme-roundtrip-qwen3-8-27b-q4-2026-10-30T03-42-03-123Z-so4g", "success": false, "includedInCapability": true, "durationMs": 344455, "timing": { "startedAt": "2026-10-30T03:42:03.124Z", "finishedAt": "2026-10-30T03:47:47.579Z", "timeToFirstArtifactMs": 131279, "timeToLastArtifactWriteMs": 333322, "timeToFirstTokenMs": 22622, "firstTurnTtftMs": 20806, "timeToFirstToolCallMs": 23741 }, "metrics": { "process": { "sampleCount": 68, "peakRssMb": 43452, "peakCpuPercent": 115, "firstAtMs": 0, "lastAtMs": 336330 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 68, "peakUsedMb": 93074, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 32275, "totalOutputTokens": 6991, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 97963, "todayTokensCached": 92150, "todayTokensOut": 5523, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 97963, "totalTokensCached": 92150, "totalTokensOut": 5523, "totalCost": 0, "medianOutputTokensPerSec": 38.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 38.3, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16313, "cachedInputTokens": 15404, "outputTokens": 805, "cost": 0, "durationMs": 31011, "outputTokensPerSec": 35.16942059123208, "at": "2026-10-30T03:47:36.446Z" }, "lastUpdated": "2026-10-30T03:47:36.446Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:47:41.044Z" } }, "source": "engine-log" }, "derived": { "durationMs": 336875, "meanTokensPerSec": 37.42, "genTokensPerSec": 37.42, "genTokens": 6991, "promptTokensPerSec": 406.01, "peakRssMb": 43452, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 97963, "output": 5523 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-productivity-craftbook-powerpoint-deck", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 2 }, "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-30T03-48-09-542Z-42p2", "success": true, "includedInCapability": true, "durationMs": 217594, "timing": { "startedAt": "2026-10-30T03:48:09.542Z", "finishedAt": "2026-10-30T03:51:47.136Z", "timeToFirstArtifactMs": 43409, "timeToLastArtifactWriteMs": 203297, "timeToFirstTokenMs": 20956, "firstTurnTtftMs": 14213, "timeToFirstToolCallMs": 28946 }, "metrics": { "process": { "sampleCount": 42, "peakRssMb": 27263, "peakCpuPercent": 104.3, "firstAtMs": 0, "lastAtMs": 205659 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 42, "peakUsedMb": 89051, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 64929, "totalOutputTokens": 6549, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 42873, "todayTokensCached": 41480, "todayTokensOut": 1204, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 42873, "totalTokensCached": 41480, "totalTokensOut": 1204, "totalCost": 0, "medianOutputTokensPerSec": 85.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 85.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12341, "cachedInputTokens": 11926, "outputTokens": 39, "cost": 0, "durationMs": 54219, "outputTokensPerSec": 76.86115235093992, "at": "2026-10-30T03:50:40.224Z" }, "lastUpdated": "2026-10-30T03:50:40.224Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T03:51:37.029Z" } }, "source": "engine-log" }, "derived": { "durationMs": 206433, "meanTokensPerSec": 72.34, "genTokensPerSec": 72.34, "genTokens": 6549, "promptTokensPerSec": 947.52, "peakRssMb": 27263, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 42873, "output": 1204 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-craftbook-powerpoint-deck", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 2 }, "trialId": "craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-30T03-52-33-429Z-u0rh", "success": true, "includedInCapability": true, "durationMs": 1888275, "timing": { "startedAt": "2026-10-30T03:52:33.430Z", "finishedAt": "2026-10-30T04:24:01.705Z", "timeToFirstArtifactMs": 211027, "timeToLastArtifactWriteMs": 1876480, "timeToFirstTokenMs": 29666, "firstTurnTtftMs": 23117, "timeToFirstToolCallMs": 45010 }, "metrics": { "process": { "sampleCount": 375, "peakRssMb": 20767, "peakCpuPercent": 121.5, "firstAtMs": 1, "lastAtMs": 1876527 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 375, "peakUsedMb": 93537, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 135441, "totalOutputTokens": 31686, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 77963, "todayTokensCached": 74600, "todayTokensOut": 8510, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 77963, "totalTokensCached": 74600, "totalTokensOut": 8510, "totalCost": 0, "medianOutputTokensPerSec": 26.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.1, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 15649, "cachedInputTokens": 15574, "outputTokens": 382, "cost": 0, "durationMs": 244172, "outputTokensPerSec": 17.910705788390956, "at": "2026-10-30T04:17:34.621Z" }, "lastUpdated": "2026-10-30T04:17:34.621Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T04:23:54.810Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1880326, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 31686, "promptTokensPerSec": null, "peakRssMb": 20767, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 77963, "output": 8510 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 522, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-craftbook-powerpoint-deck", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 2 }, "trialId": "craftbook-powerpoint-deck-qwen3-8-27b-q4-2026-10-30T04-24-27-145Z-5lbh", "success": true, "includedInCapability": true, "durationMs": 1021328, "timing": { "startedAt": "2026-10-30T04:24:27.146Z", "finishedAt": "2026-10-30T04:41:28.474Z", "timeToFirstArtifactMs": 136138, "timeToLastArtifactWriteMs": 1006845, "timeToFirstTokenMs": 39134, "firstTurnTtftMs": 32455, "timeToFirstToolCallMs": 53671 }, "metrics": { "process": { "sampleCount": 202, "peakRssMb": 49700, "peakCpuPercent": 112.5, "firstAtMs": 0, "lastAtMs": 1008365 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 202, "peakUsedMb": 103312, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 119666, "totalOutputTokens": 16365, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 76011, "todayTokensCached": 70743, "todayTokensOut": 3148, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 76011, "totalTokensCached": 70743, "totalTokensOut": 3148, "totalCost": 0, "medianOutputTokensPerSec": 34.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 34.2, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14254, "cachedInputTokens": 12994, "outputTokens": 73, "cost": 0, "durationMs": 186240, "outputTokensPerSec": 34.946519690665056, "at": "2026-10-30T04:38:20.165Z" }, "lastUpdated": "2026-10-30T04:38:20.165Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T04:41:18.346Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1010148, "meanTokensPerSec": 27.02, "genTokensPerSec": 27.02, "genTokens": 16365, "promptTokensPerSec": 351.17, "peakRssMb": 49700, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 76011, "output": 3148 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-craftbook-powerpoint-deck", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 2 }, "trialId": "craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-30T04-41-50-457Z-1glo", "success": true, "includedInCapability": true, "durationMs": 3310027, "timing": { "startedAt": "2026-10-30T04:41:50.458Z", "finishedAt": "2026-10-30T05:37:00.485Z", "timeToFirstArtifactMs": 53359, "timeToLastArtifactWriteMs": 3307167, "timeToFirstTokenMs": 22088, "firstTurnTtftMs": 15616, "timeToFirstToolCallMs": 34195 }, "metrics": { "process": { "sampleCount": 659, "peakRssMb": 13322, "peakCpuPercent": 131.9, "firstAtMs": 0, "lastAtMs": 3303673 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 659, "peakUsedMb": 131028, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 658407, "totalOutputTokens": 52246, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 27, "todayTokensIn": 816001, "todayTokensCached": 567178, "todayTokensOut": 22014, "todayCost": 0, "totalTurns": 27, "totalTokensIn": 816001, "totalTokensCached": 567178, "totalTokensOut": 22014, "totalCost": 0, "medianOutputTokensPerSec": 34.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.1, "turns": 27 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 50335, "cachedInputTokens": 50207, "outputTokens": 247, "cost": 0, "durationMs": 80460, "outputTokensPerSec": 32.25854991700652, "at": "2026-10-30T05:36:05.570Z" }, "lastUpdated": "2026-10-30T05:36:05.570Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T05:36:58.135Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3306720, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 52246, "promptTokensPerSec": null, "peakRssMb": 13322, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 816001, "output": 22014 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 893, "sampleCount": 30, "structuredSampleCount": 0, "firstTokenSampleCount": 30, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-productivity-wikipedia-research-brief", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 2 }, "trialId": "wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-30T05-37-22-994Z-2803", "success": false, "includedInCapability": true, "durationMs": 615241, "timing": { "startedAt": "2026-10-30T05:37:22.995Z", "finishedAt": "2026-10-30T05:47:38.236Z", "timeToFirstArtifactMs": 93240, "timeToLastArtifactWriteMs": 609734, "timeToFirstTokenMs": 13039, "firstTurnTtftMs": 11340, "timeToFirstToolCallMs": 57341 }, "metrics": { "process": { "sampleCount": 123, "peakRssMb": 17921, "peakCpuPercent": 110.2, "firstAtMs": 0, "lastAtMs": 612041 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 123, "peakUsedMb": 58378, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 170772, "totalOutputTokens": 6619, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 104826, "todayTokensCached": 26225, "todayTokensOut": 3122, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 104826, "totalTokensCached": 26225, "totalTokensOut": 3122, "totalCost": 0, "medianOutputTokensPerSec": 28.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.7, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 19954, "cachedInputTokens": 1690, "outputTokens": 224, "cost": 0, "durationMs": 81810, "outputTokensPerSec": 25.827265233732987, "at": "2026-10-30T05:47:32.732Z" }, "lastUpdated": "2026-10-30T05:47:32.732Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T05:47:36.675Z" } }, "source": "engine-log" }, "derived": { "durationMs": 612584, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 6619, "promptTokensPerSec": null, "peakRssMb": 17921, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 104826, "output": 3122 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 530, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-wikipedia-research-brief", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 2 }, "trialId": "wikipedia-research-brief-qwen3-8-27b-q4-2026-10-30T05-48-01-482Z-6qj4", "success": true, "includedInCapability": true, "durationMs": 385870, "timing": { "startedAt": "2026-10-30T05:48:01.483Z", "finishedAt": "2026-10-30T05:54:27.353Z", "timeToFirstArtifactMs": 312479, "timeToLastArtifactWriteMs": 381407, "timeToFirstTokenMs": 15265, "firstTurnTtftMs": 13366, "timeToFirstToolCallMs": 91305 }, "metrics": { "process": { "sampleCount": 77, "peakRssMb": 40620, "peakCpuPercent": 106.2, "firstAtMs": 0, "lastAtMs": 381218 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 77, "peakUsedMb": 64944, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 28521, "totalOutputTokens": 8875, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 37435, "todayTokensCached": 21597, "todayTokensOut": 574, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 37435, "totalTokensCached": 21597, "totalTokensOut": 574, "totalCost": 0, "medianOutputTokensPerSec": 36.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.4, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14497, "cachedInputTokens": 12419, "outputTokens": 174, "cost": 0, "durationMs": 14116, "outputTokensPerSec": 36.44426954940136, "at": "2026-10-30T05:54:22.893Z" }, "lastUpdated": "2026-10-30T05:54:22.893Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T05:54:24.313Z" } }, "source": "engine-log" }, "derived": { "durationMs": 381756, "meanTokensPerSec": 29.78, "genTokensPerSec": 29.78, "genTokens": 8875, "promptTokensPerSec": 387.36, "peakRssMb": 40620, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 37435, "output": 574 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-wikipedia-research-brief", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 2 }, "trialId": "wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-30T05-54-48-778Z-fskp", "success": false, "includedInCapability": true, "durationMs": 243725, "timing": { "startedAt": "2026-10-30T05:54:48.778Z", "finishedAt": "2026-10-30T05:58:52.503Z", "timeToFirstArtifactMs": 57269, "timeToLastArtifactWriteMs": 239803, "timeToFirstTokenMs": 12591, "firstTurnTtftMs": 10871, "timeToFirstToolCallMs": 34950 }, "metrics": { "process": { "sampleCount": 49, "peakRssMb": 10524, "peakCpuPercent": 99.6, "firstAtMs": 0, "lastAtMs": 240987 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 43833, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 45172, "totalOutputTokens": 5814, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 46596, "todayTokensCached": 25350, "todayTokensOut": 3952, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 46596, "totalTokensCached": 25350, "totalTokensOut": 3952, "totalCost": 0, "medianOutputTokensPerSec": 36.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.7, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13636, "cachedInputTokens": 1000, "outputTokens": 771, "cost": 0, "durationMs": 82870, "outputTokensPerSec": 34.93781888771293, "at": "2026-10-30T05:58:48.581Z" }, "lastUpdated": "2026-10-30T05:58:48.581Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T05:58:51.423Z" } }, "source": "engine-log" }, "derived": { "durationMs": 241526, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 5814, "promptTokensPerSec": null, "peakRssMb": 10524, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 46596, "output": 3952 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1068, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-wikipedia-research-brief", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 2 }, "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-30T05-59-13-977Z-6f7x", "success": true, "includedInCapability": true, "durationMs": 233315, "timing": { "startedAt": "2026-10-30T05:59:13.978Z", "finishedAt": "2026-10-30T06:03:07.293Z", "timeToFirstArtifactMs": 48421, "timeToLastArtifactWriteMs": 226627, "timeToFirstTokenMs": 9946, "firstTurnTtftMs": 8026, "timeToFirstToolCallMs": 26529 }, "metrics": { "process": { "sampleCount": 46, "peakRssMb": 26600, "peakCpuPercent": 122, "firstAtMs": 0, "lastAtMs": 225779 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 51258, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 37095, "totalOutputTokens": 10487, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 112785, "todayTokensCached": 104317, "todayTokensOut": 6223, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 112785, "totalTokensCached": 104317, "totalTokensOut": 6223, "totalCost": 0, "medianOutputTokensPerSec": 69.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 69.3, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 23842, "cachedInputTokens": 22491, "outputTokens": 772, "cost": 0, "durationMs": 19712, "outputTokensPerSec": 72.7109695578692, "at": "2026-10-30T06:03:00.608Z" }, "lastUpdated": "2026-10-30T06:03:00.608Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T06:03:01.374Z" } }, "source": "engine-log" }, "derived": { "durationMs": 226322, "meanTokensPerSec": 64.11, "genTokensPerSec": 64.11, "genTokens": 10487, "promptTokensPerSec": 922.53, "peakRssMb": 26600, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 112785, "output": 6223 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-productivity-craftbook-week-plan", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 2 }, "trialId": "craftbook-week-plan-qwen3-8-27b-q4-2026-10-30T06-03-29-374Z-1gcc", "success": false, "includedInCapability": true, "durationMs": 5404411, "timing": { "startedAt": "2026-10-30T06:03:29.374Z", "finishedAt": "2026-10-30T07:33:33.785Z", "timeToFirstArtifactMs": 143596, "timeToLastArtifactWriteMs": 5340471, "timeToFirstTokenMs": 47225, "firstTurnTtftMs": 40475, "timeToFirstToolCallMs": 50824 }, "metrics": { "process": { "sampleCount": 1077, "peakRssMb": 49651, "peakCpuPercent": 107.3, "firstAtMs": 0, "lastAtMs": 5399159 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1077, "peakUsedMb": 81633, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 418183, "totalOutputTokens": 81649, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 23, "todayTokensIn": 420844, "todayTokensCached": 405626, "todayTokensOut": 25398, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 420844, "totalTokensCached": 405626, "totalTokensOut": 25398, "totalCost": 0, "medianOutputTokensPerSec": 25.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.7, "turns": 23 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 23123, "cachedInputTokens": 23033, "outputTokens": 1361, "cost": 0, "durationMs": 301311, "outputTokensPerSec": 23.501912087918164, "at": "2026-10-30T07:32:29.845Z" }, "lastUpdated": "2026-10-30T07:32:29.845Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T07:33:31.415Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5400940, "meanTokensPerSec": 24.24, "genTokensPerSec": 24.24, "genTokens": 81649, "promptTokensPerSec": 410.92, "peakRssMb": 49651, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 420844, "output": 25398 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-craftbook-week-plan", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 2 }, "trialId": "craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-30T07-33-57-222Z-0akv", "success": false, "includedInCapability": true, "durationMs": 4137952, "timing": { "startedAt": "2026-10-30T07:33:57.223Z", "finishedAt": "2026-10-30T08:42:55.175Z", "timeToFirstArtifactMs": 47831, "timeToLastArtifactWriteMs": 3917251, "timeToFirstTokenMs": 24298, "firstTurnTtftMs": 17748, "timeToFirstToolCallMs": 33939 }, "metrics": { "process": { "sampleCount": 822, "peakRssMb": 13966, "peakCpuPercent": 156, "firstAtMs": 0, "lastAtMs": 4122133 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 822, "peakUsedMb": 59218, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 344767, "totalOutputTokens": 98314, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 30, "todayTokensIn": 378617, "todayTokensCached": 335948, "todayTokensOut": 26548, "todayCost": 0, "totalTurns": 30, "totalTokensIn": 378617, "totalTokensCached": 335948, "totalTokensOut": 26548, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 30 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12597, "cachedInputTokens": 0, "outputTokens": 4443, "cost": 0, "durationMs": 108041, "outputTokensPerSec": 44.2545952417632, "at": "2026-10-30T08:37:07.807Z" }, "lastUpdated": "2026-10-30T08:37:07.807Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T08:42:43.025Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4124698, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 98314, "promptTokensPerSec": null, "peakRssMb": 13966, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 378617, "output": 26548 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1190, "sampleCount": 37, "structuredSampleCount": 0, "firstTokenSampleCount": 37, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-craftbook-week-plan", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 2 }, "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-30T08-43-17-667Z-7afq", "success": true, "includedInCapability": true, "durationMs": 488665, "timing": { "startedAt": "2026-10-30T08:43:17.667Z", "finishedAt": "2026-10-30T08:51:26.332Z", "timeToFirstArtifactMs": 51306, "timeToLastArtifactWriteMs": 463304, "timeToFirstTokenMs": 23830, "firstTurnTtftMs": 17097, "timeToFirstToolCallMs": 30542 }, "metrics": { "process": { "sampleCount": 96, "peakRssMb": 30597, "peakCpuPercent": 148.1, "firstAtMs": 0, "lastAtMs": 476545 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 96, "peakUsedMb": 76875, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 139906, "totalOutputTokens": 16355, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 156036, "todayTokensCached": 147213, "todayTokensOut": 4980, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 156036, "totalTokensCached": 147213, "totalTokensOut": 4980, "totalCost": 0, "medianOutputTokensPerSec": 70.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.9, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6593, "cachedInputTokens": 5944, "outputTokens": 701, "cost": 0, "durationMs": 10586, "outputTokensPerSec": 70.3708119565237, "at": "2026-10-30T08:51:00.974Z" }, "lastUpdated": "2026-10-30T08:51:00.974Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T08:51:15.957Z" } }, "source": "engine-log" }, "derived": { "durationMs": 477319, "meanTokensPerSec": 66.69, "genTokensPerSec": 66.69, "genTokens": 16355, "promptTokensPerSec": 960.63, "peakRssMb": 30597, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 156036, "output": 4980 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-craftbook-week-plan", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 2 }, "trialId": "craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-30T08-51-49-083Z-ebq2", "success": false, "includedInCapability": true, "durationMs": 4238411, "timing": { "startedAt": "2026-10-30T08:51:49.083Z", "finishedAt": "2026-10-30T10:02:27.494Z", "timeToFirstArtifactMs": 1104057, "timeToLastArtifactWriteMs": 3847070, "timeToFirstTokenMs": 38248, "firstTurnTtftMs": 31647, "timeToFirstToolCallMs": 44739 }, "metrics": { "process": { "sampleCount": 843, "peakRssMb": 19692, "peakCpuPercent": 111, "firstAtMs": 0, "lastAtMs": 4224107 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 843, "peakUsedMb": 68254, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 270891, "totalOutputTokens": 78052, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 278403, "todayTokensCached": 269367, "todayTokensOut": 19771, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 278403, "totalTokensCached": 269367, "totalTokensOut": 19771, "totalCost": 0, "medianOutputTokensPerSec": 25.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.4, "turns": 13 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 29064, "cachedInputTokens": 28768, "outputTokens": 268, "cost": 0, "durationMs": 42267, "outputTokensPerSec": 27.43758422602941, "at": "2026-10-30T09:58:35.742Z" }, "lastUpdated": "2026-10-30T09:58:35.742Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T10:02:17.355Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4227160, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 78052, "promptTokensPerSec": null, "peakRssMb": 19692, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 278403, "output": 19771 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 531, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-productivity-craftbook-ab-test-readout", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 2 }, "trialId": "craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-30T10-02-49-828Z-y69i", "success": false, "includedInCapability": true, "durationMs": 2427317, "timing": { "startedAt": "2026-10-30T10:02:49.829Z", "finishedAt": "2026-10-30T10:43:17.146Z", "timeToFirstArtifactMs": 50905, "timeToLastArtifactWriteMs": 2325596, "timeToFirstTokenMs": 23845, "firstTurnTtftMs": 17336, "timeToFirstToolCallMs": 43611 }, "metrics": { "process": { "sampleCount": 482, "peakRssMb": 14688, "peakCpuPercent": 163.3, "firstAtMs": 0, "lastAtMs": 2413683 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 482, "peakUsedMb": 56842, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 244668, "totalOutputTokens": 65167, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 20, "todayTokensIn": 229191, "todayTokensCached": 221660, "todayTokensOut": 17534, "todayCost": 0, "totalTurns": 20, "totalTokensIn": 229191, "totalTokensCached": 221660, "totalTokensOut": 17534, "totalCost": 0, "medianOutputTokensPerSec": 36.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.1, "turns": 20 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19973, "cachedInputTokens": 19914, "outputTokens": 483, "cost": 0, "durationMs": 58797, "outputTokensPerSec": 34.91822302109004, "at": "2026-10-30T10:41:35.425Z" }, "lastUpdated": "2026-10-30T10:41:35.425Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T10:43:07.874Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2416987, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 65167, "promptTokensPerSec": null, "peakRssMb": 14688, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 229191, "output": 17534 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1096.5, "sampleCount": 28, "structuredSampleCount": 0, "firstTokenSampleCount": 28, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-craftbook-ab-test-readout", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 2 }, "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-30T10-43-39-353Z-shby", "success": false, "includedInCapability": true, "durationMs": 2414379, "timing": { "startedAt": "2026-10-30T10:43:39.353Z", "finishedAt": "2026-10-30T11:23:53.732Z", "timeToFirstArtifactMs": 30710, "timeToLastArtifactWriteMs": 2402515, "timeToFirstTokenMs": 21001, "firstTurnTtftMs": 14286, "timeToFirstToolCallMs": 27446 }, "metrics": { "process": { "sampleCount": 479, "peakRssMb": 32923, "peakCpuPercent": 140, "firstAtMs": 0, "lastAtMs": 2399306 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 479, "peakUsedMb": 73776, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 386109, "totalOutputTokens": 90189, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 31, "todayTokensIn": 373545, "todayTokensCached": 348315, "todayTokensOut": 11337, "todayCost": 0, "totalTurns": 31, "totalTokensIn": 373545, "totalTokensCached": 348315, "totalTokensOut": 11337, "totalCost": 0, "medianOutputTokensPerSec": 77.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 77.1, "turns": 31 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14731, "cachedInputTokens": 13914, "outputTokens": 155, "cost": 0, "durationMs": 33781, "outputTokensPerSec": 82.88027245005925, "at": "2026-10-30T11:12:55.244Z" }, "lastUpdated": "2026-10-30T11:12:55.244Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T11:23:44.269Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2403869, "meanTokensPerSec": 69.11, "genTokensPerSec": 69.11, "genTokens": 90189, "promptTokensPerSec": 1008.19, "peakRssMb": 32923, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 373545, "output": 11337 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-craftbook-ab-test-readout", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 2 }, "trialId": "craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-30T11-24-14-783Z-5et1", "success": true, "includedInCapability": true, "durationMs": 3445984, "timing": { "startedAt": "2026-10-30T11:24:14.783Z", "finishedAt": "2026-10-30T12:21:40.767Z", "timeToFirstArtifactMs": 117365, "timeToLastArtifactWriteMs": 3438936, "timeToFirstTokenMs": 36251, "firstTurnTtftMs": 29741, "timeToFirstToolCallMs": 37790 }, "metrics": { "process": { "sampleCount": 686, "peakRssMb": 19522, "peakCpuPercent": 106.6, "firstAtMs": 0, "lastAtMs": 3436734 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 686, "peakUsedMb": 70142, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 121321, "totalOutputTokens": 76611, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 114759, "todayTokensCached": 107066, "todayTokensOut": 19948, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 114759, "totalTokensCached": 107066, "totalTokensOut": 19948, "totalCost": 0, "medianOutputTokensPerSec": 29.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.2, "turns": 10 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4923, "cachedInputTokens": 2905, "outputTokens": 65, "cost": 0, "durationMs": 4738, "outputTokensPerSec": 38.9021361300412, "at": "2026-10-30T12:21:33.721Z" }, "lastUpdated": "2026-10-30T12:21:33.721Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T12:21:34.368Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3438518, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 76611, "promptTokensPerSec": null, "peakRssMb": 19522, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 114759, "output": 19948 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 577.5, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-craftbook-ab-test-readout", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 2 }, "trialId": "craftbook-ab-test-readout-qwen3-8-27b-q4-2026-10-30T12-22-07-365Z-0eeh", "success": true, "includedInCapability": true, "durationMs": 1132703, "timing": { "startedAt": "2026-10-30T12:22:07.365Z", "finishedAt": "2026-10-30T12:41:00.068Z", "timeToFirstArtifactMs": 78202, "timeToLastArtifactWriteMs": 1129231, "timeToFirstTokenMs": 42256, "firstTurnTtftMs": 35586, "timeToFirstToolCallMs": 46774 }, "metrics": { "process": { "sampleCount": 226, "peakRssMb": 48017, "peakCpuPercent": 105.4, "firstAtMs": 0, "lastAtMs": 1128544 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 226, "peakUsedMb": 88085, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 138062, "totalOutputTokens": 20806, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 110795, "todayTokensCached": 107193, "todayTokensOut": 6002, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 110795, "totalTokensCached": 107193, "totalTokensOut": 6002, "totalCost": 0, "medianOutputTokensPerSec": 32, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9028, "cachedInputTokens": 8516, "outputTokens": 66, "cost": 0, "durationMs": 39970, "outputTokensPerSec": 35.0990305647758, "at": "2026-10-30T12:40:04.153Z" }, "lastUpdated": "2026-10-30T12:40:04.153Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T12:40:57.470Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1129065, "meanTokensPerSec": 29.15, "genTokensPerSec": 29.15, "genTokens": 20806, "promptTokensPerSec": 395.58, "peakRssMb": 48017, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 110795, "output": 6002 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-productivity-craftbook-annotated-bibliography", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 2 }, "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-30T12-41-24-530Z-rz34", "success": true, "includedInCapability": true, "durationMs": 171487, "timing": { "startedAt": "2026-10-30T12:41:24.530Z", "finishedAt": "2026-10-30T12:44:16.017Z", "timeToFirstArtifactMs": 30186, "timeToLastArtifactWriteMs": 159907, "timeToFirstTokenMs": 18507, "firstTurnTtftMs": 11789, "timeToFirstToolCallMs": 25012 }, "metrics": { "process": { "sampleCount": 33, "peakRssMb": 25815, "peakCpuPercent": 147, "firstAtMs": 0, "lastAtMs": 160507 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 67427, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44394, "totalOutputTokens": 6477, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44363, "todayTokensCached": 41814, "todayTokensOut": 1440, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44363, "totalTokensCached": 41814, "totalTokensOut": 1440, "totalCost": 0, "medianOutputTokensPerSec": 77.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 77.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12489, "cachedInputTokens": 11494, "outputTokens": 208, "cost": 0, "durationMs": 28372, "outputTokensPerSec": 83.72644991358379, "at": "2026-10-30T12:44:04.437Z" }, "lastUpdated": "2026-10-30T12:44:04.437Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T12:44:06.613Z" } }, "source": "engine-log" }, "derived": { "durationMs": 161031, "meanTokensPerSec": 72.76, "genTokensPerSec": 72.76, "genTokens": 6477, "promptTokensPerSec": 1065.25, "peakRssMb": 25815, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44363, "output": 1440 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-productivity-craftbook-annotated-bibliography", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 2 }, "trialId": "craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-30T12-44-37-552Z-8w4x", "success": true, "includedInCapability": true, "durationMs": 697204, "timing": { "startedAt": "2026-10-30T12:44:37.552Z", "finishedAt": "2026-10-30T12:56:14.756Z", "timeToFirstArtifactMs": 72444, "timeToLastArtifactWriteMs": 686093, "timeToFirstTokenMs": 26643, "firstTurnTtftMs": 20123, "timeToFirstToolCallMs": 72443 }, "metrics": { "process": { "sampleCount": 138, "peakRssMb": 19056, "peakCpuPercent": 120.3, "firstAtMs": 0, "lastAtMs": 687196 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 138, "peakUsedMb": 70244, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 45084, "totalOutputTokens": 14644, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44940, "todayTokensCached": 36475, "todayTokensOut": 6392, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44940, "totalTokensCached": 36475, "totalTokensOut": 6392, "totalCost": 0, "medianOutputTokensPerSec": 26.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.5, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12598, "cachedInputTokens": 12543, "outputTokens": 827, "cost": 0, "durationMs": 104427, "outputTokensPerSec": 22.766639754806487, "at": "2026-10-30T12:56:03.645Z" }, "lastUpdated": "2026-10-30T12:56:03.645Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T12:56:06.342Z" } }, "source": "engine-log" }, "derived": { "durationMs": 687716, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 14644, "promptTokensPerSec": null, "peakRssMb": 19056, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 44940, "output": 6392 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 560, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-craftbook-annotated-bibliography", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 2 }, "trialId": "craftbook-annotated-bibliography-qwen3-8-27b-q4-2026-10-30T12-56-35-687Z-euli", "success": true, "includedInCapability": true, "durationMs": 366181, "timing": { "startedAt": "2026-10-30T12:56:35.689Z", "finishedAt": "2026-10-30T13:02:41.870Z", "timeToFirstArtifactMs": 82903, "timeToLastArtifactWriteMs": 361045, "timeToFirstTokenMs": 31484, "firstTurnTtftMs": 24805, "timeToFirstToolCallMs": 59027 }, "metrics": { "process": { "sampleCount": 73, "peakRssMb": 44206, "peakCpuPercent": 145.3, "firstAtMs": 1, "lastAtMs": 361159 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 73, "peakUsedMb": 83122, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44135, "totalOutputTokens": 6260, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44305, "todayTokensCached": 42336, "todayTokensOut": 3631, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44305, "totalTokensCached": 42336, "totalTokensOut": 3631, "totalCost": 0, "medianOutputTokensPerSec": 26.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.7, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12607, "cachedInputTokens": 12132, "outputTokens": 1222, "cost": 0, "durationMs": 89464, "outputTokensPerSec": 25.581833772051656, "at": "2026-10-30T13:02:36.734Z" }, "lastUpdated": "2026-10-30T13:02:36.734Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T13:02:38.419Z" } }, "source": "engine-log" }, "derived": { "durationMs": 361678, "meanTokensPerSec": 27.75, "genTokensPerSec": 27.75, "genTokens": 6260, "promptTokensPerSec": 412.16, "peakRssMb": 44206, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44305, "output": 3631 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-craftbook-annotated-bibliography", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 2 }, "trialId": "craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-30T13-03-04-848Z-d9qz", "success": true, "includedInCapability": true, "durationMs": 189798, "timing": { "startedAt": "2026-10-30T13:03:04.848Z", "finishedAt": "2026-10-30T13:06:14.646Z", "timeToFirstArtifactMs": 43686, "timeToLastArtifactWriteMs": 185588, "timeToFirstTokenMs": 20246, "firstTurnTtftMs": 13747, "timeToFirstToolCallMs": 34795 }, "metrics": { "process": { "sampleCount": 38, "peakRssMb": 11412, "peakCpuPercent": 166.6, "firstAtMs": 1, "lastAtMs": 185614 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 38, "peakUsedMb": 54227, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 42571, "totalOutputTokens": 3899, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 42427, "todayTokensCached": 39668, "todayTokensOut": 1265, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 42427, "totalTokensCached": 39668, "totalTokensOut": 1265, "totalCost": 0, "medianOutputTokensPerSec": 39.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10697, "cachedInputTokens": 9828, "outputTokens": 85, "cost": 0, "durationMs": 24099, "outputTokensPerSec": 42.590933220696286, "at": "2026-10-30T13:06:10.436Z" }, "lastUpdated": "2026-10-30T13:06:10.436Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T13:06:12.051Z" } }, "source": "engine-log" }, "derived": { "durationMs": 186139, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 3899, "promptTokensPerSec": null, "peakRssMb": 11412, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 42427, "output": 1265 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1238, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-productivity-craftbook-research-to-document", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 2 }, "trialId": "craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-30T13-06-37-068Z-br3m", "success": true, "includedInCapability": true, "durationMs": 950316, "timing": { "startedAt": "2026-10-30T13:06:37.069Z", "finishedAt": "2026-10-30T13:22:27.385Z", "timeToFirstArtifactMs": 52191, "timeToLastArtifactWriteMs": 157216, "timeToFirstTokenMs": 26533, "firstTurnTtftMs": 20024, "timeToFirstToolCallMs": 28728 }, "metrics": { "process": { "sampleCount": 189, "peakRssMb": 19877, "peakCpuPercent": 148.5, "firstAtMs": 0, "lastAtMs": 943061 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 189, "peakUsedMb": 75874, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 127317, "totalOutputTokens": 4591, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 98582, "todayTokensCached": 93092, "todayTokensOut": 1765, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 98582, "totalTokensCached": 93092, "totalTokensOut": 1765, "totalCost": 0, "medianOutputTokensPerSec": 32.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.1, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 36329, "cachedInputTokens": 32117, "outputTokens": 77, "cost": 0, "durationMs": 409562, "outputTokensPerSec": 25.234273732705894, "at": "2026-10-30T13:22:19.002Z" }, "lastUpdated": "2026-10-30T13:22:19.002Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T13:22:21.987Z" } }, "source": "engine-log" }, "derived": { "durationMs": 943851, "meanTokensPerSec": 21, "genTokensPerSec": 21, "genTokens": 4591, "promptTokensPerSec": null, "peakRssMb": 19877, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 98582, "output": 1765 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 588, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-productivity-craftbook-research-to-document", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 2 }, "trialId": "craftbook-research-to-document-qwen3-8-27b-q4-2026-10-30T13-22-50-797Z-5t0t", "success": true, "includedInCapability": true, "durationMs": 651996, "timing": { "startedAt": "2026-10-30T13:22:50.797Z", "finishedAt": "2026-10-30T13:33:42.793Z", "timeToFirstArtifactMs": 63943, "timeToLastArtifactWriteMs": 187866, "timeToFirstTokenMs": 30617, "firstTurnTtftMs": 23953, "timeToFirstToolCallMs": 39374 }, "metrics": { "process": { "sampleCount": 129, "peakRssMb": 47951, "peakCpuPercent": 121.9, "firstAtMs": 0, "lastAtMs": 642018 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 129, "peakUsedMb": 88274, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 134609, "totalOutputTokens": 5703, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 103501, "todayTokensCached": 101416, "todayTokensOut": 1801, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 103501, "totalTokensCached": 101416, "totalTokensOut": 1801, "totalCost": 0, "medianOutputTokensPerSec": 31.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.4, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 37450, "cachedInputTokens": 36748, "outputTokens": 74, "cost": 0, "durationMs": 197779, "outputTokensPerSec": 31.58140192871515, "at": "2026-10-30T13:33:30.871Z" }, "lastUpdated": "2026-10-30T13:33:30.871Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T13:33:34.535Z" } }, "source": "engine-log" }, "derived": { "durationMs": 642794, "meanTokensPerSec": 27.13, "genTokensPerSec": 27.13, "genTokens": 5703, "promptTokensPerSec": 342.75, "peakRssMb": 47951, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 103501, "output": 1801 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-productivity-craftbook-research-to-document", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 2 }, "trialId": "craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-30T13-34-05-240Z-ibbe", "success": true, "includedInCapability": true, "durationMs": 2000693, "timing": { "startedAt": "2026-10-30T13:34:05.241Z", "finishedAt": "2026-10-30T14:07:25.934Z", "timeToFirstArtifactMs": 31746, "timeToLastArtifactWriteMs": 1437186, "timeToFirstTokenMs": 20022, "firstTurnTtftMs": 13500, "timeToFirstToolCallMs": 25287 }, "metrics": { "process": { "sampleCount": 398, "peakRssMb": 13796, "peakCpuPercent": 126.6, "firstAtMs": 0, "lastAtMs": 1992583 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 398, "peakUsedMb": 78243, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 223350, "totalOutputTokens": 43717, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 214899, "todayTokensCached": 208671, "todayTokensOut": 11926, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 214899, "totalTokensCached": 208671, "totalTokensOut": 11926, "totalCost": 0, "medianOutputTokensPerSec": 39.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.2, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19628, "cachedInputTokens": 17010, "outputTokens": 35, "cost": 0, "durationMs": 113569, "outputTokensPerSec": 45.309241447319735, "at": "2026-10-30T14:07:16.554Z" }, "lastUpdated": "2026-10-30T14:07:16.554Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:07:21.692Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1995391, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 43717, "promptTokensPerSec": null, "peakRssMb": 13796, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 214899, "output": 11926 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1171, "sampleCount": 14, "structuredSampleCount": 0, "firstTokenSampleCount": 14, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-productivity-craftbook-research-to-document", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 2 }, "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-30T14-07-52-001Z-6ho9", "success": true, "includedInCapability": true, "durationMs": 867882, "timing": { "startedAt": "2026-10-30T14:07:52.002Z", "finishedAt": "2026-10-30T14:22:19.884Z", "timeToFirstArtifactMs": 26676, "timeToLastArtifactWriteMs": 677569, "timeToFirstTokenMs": 17739, "firstTurnTtftMs": 10939, "timeToFirstToolCallMs": 20204 }, "metrics": { "process": { "sampleCount": 173, "peakRssMb": 33829, "peakCpuPercent": 114.6, "firstAtMs": 0, "lastAtMs": 862926 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 173, "peakUsedMb": 81205, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 261654, "totalOutputTokens": 26648, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 328555, "todayTokensCached": 320349, "todayTokensOut": 4864, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 328555, "totalTokensCached": 320349, "totalTokensOut": 4864, "totalCost": 0, "medianOutputTokensPerSec": 78, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 78, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 21678, "cachedInputTokens": 21378, "outputTokens": 35, "cost": 0, "durationMs": 36620, "outputTokensPerSec": 88.95423827700873, "at": "2026-10-30T14:22:16.056Z" }, "lastUpdated": "2026-10-30T14:22:16.056Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:22:17.252Z" } }, "source": "engine-log" }, "derived": { "durationMs": 864207, "meanTokensPerSec": 63.24, "genTokensPerSec": 63.24, "genTokens": 26648, "promptTokensPerSec": 1024.16, "peakRssMb": 33829, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 328555, "output": 4864 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-developer-fix-squisq-bugs", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 2 }, "trialId": "fix-squisq-bugs-qwen3-8-27b-q4-2026-10-30T14-22-40-806Z-160h", "success": true, "includedInCapability": true, "durationMs": 246469, "timing": { "startedAt": "2026-10-30T14:22:40.809Z", "finishedAt": "2026-10-30T14:26:47.278Z", "timeToFirstArtifactMs": 42150, "timeToLastArtifactWriteMs": 238910, "timeToFirstTokenMs": 13957, "firstTurnTtftMs": 12179, "timeToFirstToolCallMs": 16104 }, "metrics": { "process": { "sampleCount": 49, "peakRssMb": 44874, "peakCpuPercent": 96.7, "firstAtMs": 0, "lastAtMs": 240811 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 80818, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 43424, "totalOutputTokens": 5986, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 54974, "todayTokensCached": 22799, "todayTokensOut": 3279, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 54974, "totalTokensCached": 22799, "totalTokensOut": 3279, "totalCost": 0, "medianOutputTokensPerSec": 39.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 39.1, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16515, "cachedInputTokens": 13137, "outputTokens": 685, "cost": 0, "durationMs": 26810, "outputTokensPerSec": 37.51423237324686, "at": "2026-10-30T14:26:39.722Z" }, "lastUpdated": "2026-10-30T14:26:39.722Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:26:43.708Z" } }, "source": "engine-log" }, "derived": { "durationMs": 241853, "meanTokensPerSec": 45.01, "genTokensPerSec": 45.01, "genTokens": 5986, "promptTokensPerSec": 483.49, "peakRssMb": 44874, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 54974, "output": 3279 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-fix-squisq-bugs", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 2 }, "trialId": "fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-30T14-27-12-050Z-yaub", "success": true, "includedInCapability": true, "durationMs": 264261, "timing": { "startedAt": "2026-10-30T14:27:12.050Z", "finishedAt": "2026-10-30T14:31:36.311Z", "timeToFirstArtifactMs": 34533, "timeToLastArtifactWriteMs": 259023, "timeToFirstTokenMs": 12342, "firstTurnTtftMs": 10698, "timeToFirstToolCallMs": 24915 }, "metrics": { "process": { "sampleCount": 53, "peakRssMb": 10539, "peakCpuPercent": 119.4, "firstAtMs": 0, "lastAtMs": 260937 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 53, "peakUsedMb": 54767, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 103100, "totalOutputTokens": 4750, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 76707, "todayTokensCached": 14171, "todayTokensOut": 1744, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 76707, "totalTokensCached": 14171, "totalTokensOut": 1744, "totalCost": 0, "medianOutputTokensPerSec": 39.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.2, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19470, "cachedInputTokens": 1301, "outputTokens": 403, "cost": 0, "durationMs": 41659, "outputTokensPerSec": 39.157513757254875, "at": "2026-10-30T14:31:31.075Z" }, "lastUpdated": "2026-10-30T14:31:31.075Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:31:35.098Z" } }, "source": "engine-log" }, "derived": { "durationMs": 261981, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 4750, "promptTokensPerSec": null, "peakRssMb": 10539, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 76707, "output": 1744 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1090, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-fix-squisq-bugs", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 2 }, "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-30T14-31-58-636Z-0mif", "success": true, "includedInCapability": true, "durationMs": 239529, "timing": { "startedAt": "2026-10-30T14:31:58.637Z", "finishedAt": "2026-10-30T14:35:58.166Z", "timeToFirstArtifactMs": 24597, "timeToLastArtifactWriteMs": 234480, "timeToFirstTokenMs": 9554, "firstTurnTtftMs": 7819, "timeToFirstToolCallMs": 14340 }, "metrics": { "process": { "sampleCount": 48, "peakRssMb": 27149, "peakCpuPercent": 121.8, "firstAtMs": 0, "lastAtMs": 235806 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 48, "peakUsedMb": 61705, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 61791, "totalOutputTokens": 15426, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 78019, "todayTokensCached": 32637, "todayTokensOut": 12980, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 78019, "totalTokensCached": 32637, "totalTokensOut": 12980, "totalCost": 0, "medianOutputTokensPerSec": 94.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 94.5, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17993, "outputTokens": 683, "cost": 0, "durationMs": 37174, "outputTokensPerSec": 85.94066213121754, "at": "2026-10-30T14:35:53.117Z" }, "lastUpdated": "2026-10-30T14:35:53.117Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:35:56.666Z" } }, "source": "engine-log" }, "derived": { "durationMs": 237086, "meanTokensPerSec": 97.17, "genTokensPerSec": 97.17, "genTokens": 15426, "promptTokensPerSec": 1019.02, "peakRssMb": 27149, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 78019, "output": 12980 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-fix-squisq-bugs", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 2 }, "trialId": "fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-30T14-36-19-319Z-hj68", "success": true, "includedInCapability": true, "durationMs": 103159, "timing": { "startedAt": "2026-10-30T14:36:19.320Z", "finishedAt": "2026-10-30T14:38:02.479Z", "timeToFirstArtifactMs": 99458, "timeToLastArtifactWriteMs": 99484, "timeToFirstTokenMs": 15054, "firstTurnTtftMs": 13401, "timeToFirstToolCallMs": 17458 }, "metrics": { "process": { "sampleCount": 21, "peakRssMb": 17987, "peakCpuPercent": 111.1, "firstAtMs": 0, "lastAtMs": 100313 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 71672, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 7631, "totalOutputTokens": 2772, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:38:01.477Z" } }, "source": "engine-log" }, "derived": { "durationMs": 101087, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 2772, "promptTokensPerSec": null, "peakRssMb": 17987, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 567, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-developer-dev-craftbook-routing", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 2 }, "trialId": "dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-30T14-38-23-678Z-zz8i", "success": false, "includedInCapability": true, "durationMs": 723025, "timing": { "startedAt": "2026-10-30T14:38:23.678Z", "finishedAt": "2026-10-30T14:50:26.703Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 15943, "firstTurnTtftMs": 14236, "timeToFirstToolCallMs": 37516 }, "metrics": { "process": { "sampleCount": 144, "peakRssMb": 11404, "peakCpuPercent": 99.9, "firstAtMs": 0, "lastAtMs": 718546 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 144, "peakUsedMb": 55361, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 10807, "totalOutputTokens": 9720, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 21074, "todayTokensCached": 20676, "todayTokensOut": 8276, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 21074, "totalTokensCached": 20676, "totalTokensOut": 8276, "totalCost": 0, "medianOutputTokensPerSec": 42.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.4, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10727, "cachedInputTokens": 10479, "outputTokens": 84, "cost": 0, "durationMs": 28071, "outputTokensPerSec": 42.87062662140663, "at": "2026-10-30T14:42:47.131Z" }, "lastUpdated": "2026-10-30T14:42:47.131Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:50:26.587Z" } }, "source": "engine-log" }, "derived": { "durationMs": 721849, "meanTokensPerSec": 43, "genTokensPerSec": 43, "genTokens": 9720, "promptTokensPerSec": null, "peakRssMb": 11404, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21074, "output": 8276 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 850, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-dev-craftbook-routing", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 2 }, "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-30T14-50-51-148Z-tjmy", "success": false, "includedInCapability": true, "durationMs": 430734, "timing": { "startedAt": "2026-10-30T14:50:51.150Z", "finishedAt": "2026-10-30T14:58:01.884Z", "timeToFirstArtifactMs": 90982, "timeToLastArtifactWriteMs": 426111, "timeToFirstTokenMs": 12260, "firstTurnTtftMs": 10371, "timeToFirstToolCallMs": 20522 }, "metrics": { "process": { "sampleCount": 86, "peakRssMb": 26375, "peakCpuPercent": 115.7, "firstAtMs": 0, "lastAtMs": 426412 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 86, "peakUsedMb": 67904, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 167082, "totalOutputTokens": 11194, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 104540, "todayTokensCached": 11132, "todayTokensOut": 4288, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 104540, "totalTokensCached": 11132, "totalTokensOut": 4288, "totalCost": 0, "medianOutputTokensPerSec": 74.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 74.1, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 21073, "outputTokens": 1675, "cost": 0, "durationMs": 62387, "outputTokensPerSec": 46.44359425477757, "at": "2026-10-30T14:57:57.266Z" }, "lastUpdated": "2026-10-30T14:57:57.266Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T14:58:00.631Z" } }, "source": "engine-log" }, "derived": { "durationMs": 428445, "meanTokensPerSec": 68.09, "genTokensPerSec": 68.09, "genTokens": 11194, "promptTokensPerSec": 957.75, "peakRssMb": 26375, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 104540, "output": 4288 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-dev-craftbook-routing", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 2 }, "trialId": "dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-30T14-58-24-158Z-43sc", "success": true, "includedInCapability": true, "durationMs": 118723, "timing": { "startedAt": "2026-10-30T14:58:24.158Z", "finishedAt": "2026-10-30T15:00:22.881Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 32726, "firstTurnTtftMs": 30995, "timeToFirstToolCallMs": 53014 }, "metrics": { "process": { "sampleCount": 24, "peakRssMb": 18048, "peakCpuPercent": 122.1, "firstAtMs": 0, "lastAtMs": 115368 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 72362, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13237, "totalOutputTokens": 1769, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:00:21.620Z" } }, "source": "engine-log" }, "derived": { "durationMs": 116392, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 1769, "promptTokensPerSec": null, "peakRssMb": 18048, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 487, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-dev-craftbook-routing", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 2 }, "trialId": "dev-craftbook-routing-qwen3-8-27b-q4-2026-10-30T15-00-44-530Z-b6xd", "success": true, "includedInCapability": true, "durationMs": 90492, "timing": { "startedAt": "2026-10-30T15:00:44.531Z", "finishedAt": "2026-10-30T15:02:15.023Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 33661, "firstTurnTtftMs": 31783, "timeToFirstToolCallMs": 54667 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 37807, "peakCpuPercent": 97.2, "firstAtMs": 0, "lastAtMs": 85257 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 75807, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 14569, "totalOutputTokens": 1487, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:02:11.602Z" } }, "source": "engine-log" }, "derived": { "durationMs": 86029, "meanTokensPerSec": 33.23, "genTokensPerSec": 33.23, "genTokens": 1487, "promptTokensPerSec": 440.56, "peakRssMb": 37807, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-developer-interface-contract", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "interface-contract", "repetition": 2 }, "trialId": "interface-contract-gemma4-12b-q4-2026-10-30T15-02-40-942Z-1ala", "success": true, "includedInCapability": true, "durationMs": 56331, "timing": { "startedAt": "2026-10-30T15:02:40.942Z", "finishedAt": "2026-10-30T15:03:37.273Z", "timeToFirstArtifactMs": 38004, "timeToLastArtifactWriteMs": 46180, "timeToFirstTokenMs": 11630, "firstTurnTtftMs": 9495, "timeToFirstToolCallMs": 20534 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 20587, "peakCpuPercent": 105.3, "firstAtMs": 0, "lastAtMs": 45142 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 59898, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15496, "totalOutputTokens": 1896, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7330, "todayTokensCached": 7156, "todayTokensOut": 24, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7330, "totalTokensCached": 7156, "totalTokensOut": 24, "totalCost": 0, "medianOutputTokensPerSec": 76, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 76, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7330, "cachedInputTokens": 7156, "outputTokens": 24, "cost": 0, "durationMs": 20036, "outputTokensPerSec": 75.95948387182003, "at": "2026-10-30T15:03:03.113Z" }, "lastUpdated": "2026-10-30T15:03:03.113Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:03:27.864Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45680, "meanTokensPerSec": 93.63, "genTokensPerSec": 93.63, "genTokens": 1896, "promptTokensPerSec": 1188.73, "peakRssMb": 20587, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7330, "output": 24 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-interface-contract", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "interface-contract", "repetition": 2 }, "trialId": "interface-contract-mlx-qwen3-8-27b-q4-2026-10-30T15-04-01-145Z-6gdo", "success": true, "includedInCapability": true, "durationMs": 103304, "timing": { "startedAt": "2026-10-30T15:04:01.146Z", "finishedAt": "2026-10-30T15:05:44.450Z", "timeToFirstArtifactMs": 96806, "timeToLastArtifactWriteMs": 96822, "timeToFirstTokenMs": 24148, "firstTurnTtftMs": 22461, "timeToFirstToolCallMs": 37248 }, "metrics": { "process": { "sampleCount": 21, "peakRssMb": 18696, "peakCpuPercent": 122.9, "firstAtMs": 0, "lastAtMs": 100330 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 72286, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17204, "totalOutputTokens": 2322, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11939, "todayTokensCached": 11787, "todayTokensOut": 336, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11939, "totalTokensCached": 11787, "totalTokensOut": 336, "totalCost": 0, "medianOutputTokensPerSec": 35.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 35.9, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11939, "cachedInputTokens": 11787, "outputTokens": 336, "cost": 0, "durationMs": 45348, "outputTokensPerSec": 35.911243859426115, "at": "2026-10-30T15:04:48.181Z" }, "lastUpdated": "2026-10-30T15:04:48.181Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:05:43.317Z" } }, "source": "engine-log" }, "derived": { "durationMs": 101106, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 2322, "promptTokensPerSec": null, "peakRssMb": 18696, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11939, "output": 336 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 735.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-interface-contract", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "interface-contract", "repetition": 2 }, "trialId": "interface-contract-qwen3-8-27b-q4-2026-10-30T15-06-08-081Z-ul3t", "success": true, "includedInCapability": true, "durationMs": 69602, "timing": { "startedAt": "2026-10-30T15:06:08.081Z", "finishedAt": "2026-10-30T15:07:17.683Z", "timeToFirstArtifactMs": 53870, "timeToLastArtifactWriteMs": 64764, "timeToFirstTokenMs": 21338, "firstTurnTtftMs": 19488, "timeToFirstToolCallMs": 30068 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 39038, "peakCpuPercent": 118.4, "firstAtMs": 0, "lastAtMs": 65203 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 78874, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15014, "totalOutputTokens": 1172, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7630, "todayTokensCached": 7098, "todayTokensOut": 100, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7630, "totalTokensCached": 7098, "totalTokensOut": 100, "totalCost": 0, "medianOutputTokensPerSec": 34.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 34.7, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7630, "cachedInputTokens": 7098, "outputTokens": 100, "cost": 0, "durationMs": 32373, "outputTokensPerSec": 34.73725651463912, "at": "2026-10-30T15:06:42.304Z" }, "lastUpdated": "2026-10-30T15:06:42.304Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:07:14.851Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65723, "meanTokensPerSec": 46.36, "genTokensPerSec": 46.36, "genTokens": 1172, "promptTokensPerSec": 557.47, "peakRssMb": 39038, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7630, "output": 100 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-interface-contract", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "interface-contract", "repetition": 2 }, "trialId": "interface-contract-mlx-gemma4-12b-q4-2026-10-30T15-07-39-504Z-xf4f", "success": true, "includedInCapability": true, "durationMs": 77987, "timing": { "startedAt": "2026-10-30T15:07:39.505Z", "finishedAt": "2026-10-30T15:08:57.492Z", "timeToFirstArtifactMs": 68051, "timeToLastArtifactWriteMs": 75697, "timeToFirstTokenMs": 15343, "firstTurnTtftMs": 13649, "timeToFirstToolCallMs": 51770 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 10633, "peakCpuPercent": 121, "firstAtMs": 0, "lastAtMs": 75253 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 55663, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18328, "totalOutputTokens": 1844, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 10561, "todayTokensCached": 10383, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 10561, "totalTokensCached": 10383, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 45.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 45.8, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10561, "cachedInputTokens": 10383, "outputTokens": 23, "cost": 0, "durationMs": 51573, "outputTokensPerSec": 45.75403027291955, "at": "2026-10-30T15:08:32.772Z" }, "lastUpdated": "2026-10-30T15:08:32.772Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:08:56.344Z" } }, "source": "engine-log" }, "derived": { "durationMs": 75777, "meanTokensPerSec": 46, "genTokensPerSec": 46, "genTokens": 1844, "promptTokensPerSec": null, "peakRssMb": 10633, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 10561, "output": 23 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1534.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-developer-craftbook-code-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 2 }, "trialId": "craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-30T15-09-19-166Z-t6c7", "success": true, "includedInCapability": true, "durationMs": 290528, "timing": { "startedAt": "2026-10-30T15:09:19.166Z", "finishedAt": "2026-10-30T15:14:09.694Z", "timeToFirstArtifactMs": 280242, "timeToLastArtifactWriteMs": 280242, "timeToFirstTokenMs": 23967, "firstTurnTtftMs": 17419, "timeToFirstToolCallMs": 25032 }, "metrics": { "process": { "sampleCount": 57, "peakRssMb": 17941, "peakCpuPercent": 120.2, "firstAtMs": 1, "lastAtMs": 280835 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 57, "peakUsedMb": 71261, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 21418, "totalOutputTokens": 6615, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 21338, "todayTokensCached": 20992, "todayTokensOut": 2058, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 21338, "totalTokensCached": 20992, "totalTokensOut": 2058, "totalCost": 0, "medianOutputTokensPerSec": 31, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11309, "cachedInputTokens": 11049, "outputTokens": 1912, "cost": 0, "durationMs": 213607, "outputTokensPerSec": 26.767183252535563, "at": "2026-10-30T15:13:59.408Z" }, "lastUpdated": "2026-10-30T15:13:59.408Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:14:01.581Z" } }, "source": "engine-log" }, "derived": { "durationMs": 281356, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 6615, "promptTokensPerSec": null, "peakRssMb": 17941, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21338, "output": 2058 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 675, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-craftbook-code-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 2 }, "trialId": "craftbook-code-review-qwen3-8-27b-q4-2026-10-30T15-14-32-017Z-aqgq", "success": true, "includedInCapability": true, "durationMs": 300859, "timing": { "startedAt": "2026-10-30T15:14:32.018Z", "finishedAt": "2026-10-30T15:19:32.877Z", "timeToFirstArtifactMs": 209951, "timeToLastArtifactWriteMs": 209951, "timeToFirstTokenMs": 30080, "firstTurnTtftMs": 23381, "timeToFirstToolCallMs": 31311 }, "metrics": { "process": { "sampleCount": 60, "peakRssMb": 42354, "peakCpuPercent": 173.1, "firstAtMs": 0, "lastAtMs": 295926 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 60, "peakUsedMb": 82477, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47834, "totalOutputTokens": 5106, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 33994, "todayTokensCached": 21807, "todayTokensOut": 3474, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 33994, "totalTokensCached": 21807, "totalTokensOut": 3474, "totalCost": 0, "medianOutputTokensPerSec": 32.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.5, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12085, "cachedInputTokens": 11753, "outputTokens": 60, "cost": 0, "durationMs": 45591, "outputTokensPerSec": 37.49795826795024, "at": "2026-10-30T15:18:49.241Z" }, "lastUpdated": "2026-10-30T15:18:49.241Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:19:29.517Z" } }, "source": "engine-log" }, "derived": { "durationMs": 296448, "meanTokensPerSec": 32.3, "genTokensPerSec": 32.3, "genTokens": 5106, "promptTokensPerSec": 418.47, "peakRssMb": 42354, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 33994, "output": 3474 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-craftbook-code-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 2 }, "trialId": "craftbook-code-review-mlx-gemma4-12b-q4-2026-10-30T15-19-57-027Z-3y0l", "success": true, "includedInCapability": true, "durationMs": 248990, "timing": { "startedAt": "2026-10-30T15:19:57.028Z", "finishedAt": "2026-10-30T15:24:06.018Z", "timeToFirstArtifactMs": 195595, "timeToLastArtifactWriteMs": 195595, "timeToFirstTokenMs": 19737, "firstTurnTtftMs": 13193, "timeToFirstToolCallMs": 25032 }, "metrics": { "process": { "sampleCount": 50, "peakRssMb": 10898, "peakCpuPercent": 109.4, "firstAtMs": 0, "lastAtMs": 245859 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 50, "peakUsedMb": 54611, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 36477, "totalOutputTokens": 6997, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 31479, "todayTokensCached": 20339, "todayTokensOut": 6063, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 31479, "totalTokensCached": 20339, "totalTokensOut": 6063, "totalCost": 0, "medianOutputTokensPerSec": 40.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.3, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10829, "cachedInputTokens": 10640, "outputTokens": 33, "cost": 0, "durationMs": 37073, "outputTokensPerSec": 40.27625738430684, "at": "2026-10-30T15:23:50.699Z" }, "lastUpdated": "2026-10-30T15:23:50.699Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:24:04.476Z" } }, "source": "engine-log" }, "derived": { "durationMs": 246381, "meanTokensPerSec": 40, "genTokensPerSec": 40, "genTokens": 6997, "promptTokensPerSec": null, "peakRssMb": 10898, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 31479, "output": 6063 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1239.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-craftbook-code-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 2 }, "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-30T15-24-28-163Z-dv5h", "success": true, "includedInCapability": true, "durationMs": 134188, "timing": { "startedAt": "2026-10-30T15:24:28.164Z", "finishedAt": "2026-10-30T15:26:42.352Z", "timeToFirstArtifactMs": 76301, "timeToLastArtifactWriteMs": 76301, "timeToFirstTokenMs": 17297, "firstTurnTtftMs": 10548, "timeToFirstToolCallMs": 19738 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 19547, "peakCpuPercent": 117.1, "firstAtMs": 0, "lastAtMs": 125408 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 61473, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 66797, "totalOutputTokens": 3166, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 32460, "todayTokensCached": 20908, "todayTokensOut": 601, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 32460, "totalTokensCached": 20908, "totalTokensOut": 601, "totalCost": 0, "medianOutputTokensPerSec": 92.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 92.9, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11410, "cachedInputTokens": 11195, "outputTokens": 33, "cost": 0, "durationMs": 23970, "outputTokensPerSec": 95.4241154631797, "at": "2026-10-30T15:26:13.830Z" }, "lastUpdated": "2026-10-30T15:26:13.830Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:26:35.142Z" } }, "source": "engine-log" }, "derived": { "durationMs": 125928, "meanTokensPerSec": 79.23, "genTokensPerSec": 79.23, "genTokens": 3166, "promptTokensPerSec": 1112.79, "peakRssMb": 19547, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32460, "output": 601 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-developer-craftbook-api-contract-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 2 }, "trialId": "craftbook-api-contract-review-qwen3-8-27b-q4-2026-10-30T15-27-04-335Z-dg3w", "success": true, "includedInCapability": true, "durationMs": 1206447, "timing": { "startedAt": "2026-10-30T15:27:04.336Z", "finishedAt": "2026-10-30T15:47:10.783Z", "timeToFirstArtifactMs": 238848, "timeToLastArtifactWriteMs": 1202001, "timeToFirstTokenMs": 30644, "firstTurnTtftMs": 23948, "timeToFirstToolCallMs": 36022 }, "metrics": { "process": { "sampleCount": 241, "peakRssMb": 44327, "peakCpuPercent": 146, "firstAtMs": 1, "lastAtMs": 1203750 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 241, "peakUsedMb": 86446, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 93006, "totalOutputTokens": 25661, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 74330, "todayTokensCached": 62011, "todayTokensOut": 16060, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 74330, "totalTokensCached": 62011, "totalTokensOut": 16060, "totalCost": 0, "medianOutputTokensPerSec": 30.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.8, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 26918, "cachedInputTokens": 26307, "outputTokens": 418, "cost": 0, "durationMs": 229774, "outputTokensPerSec": 32.66830995316445, "at": "2026-10-30T15:47:06.337Z" }, "lastUpdated": "2026-10-30T15:47:06.337Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T15:47:09.679Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1204296, "meanTokensPerSec": 27.38, "genTokensPerSec": 27.38, "genTokens": 25661, "promptTokensPerSec": 402.69, "peakRssMb": 44327, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 74330, "output": 16060 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-craftbook-api-contract-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 2 }, "trialId": "craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-30T15-47-33-441Z-51vg", "success": true, "includedInCapability": true, "durationMs": 1390456, "timing": { "startedAt": "2026-10-30T15:47:33.441Z", "finishedAt": "2026-10-30T16:10:43.897Z", "timeToFirstArtifactMs": 97440, "timeToLastArtifactWriteMs": 1344047, "timeToFirstTokenMs": 20636, "firstTurnTtftMs": 14115, "timeToFirstToolCallMs": 45238 }, "metrics": { "process": { "sampleCount": 277, "peakRssMb": 11972, "peakCpuPercent": 132, "firstAtMs": 0, "lastAtMs": 1385045 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 277, "peakUsedMb": 55828, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 213073, "totalOutputTokens": 34048, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 297411, "todayTokensCached": 274856, "todayTokensOut": 7049, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 297411, "totalTokensCached": 274856, "totalTokensOut": 7049, "totalCost": 0, "medianOutputTokensPerSec": 37.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.3, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8842, "cachedInputTokens": 8679, "outputTokens": 35, "cost": 0, "durationMs": 28444, "outputTokensPerSec": 36.65027286884311, "at": "2026-10-30T16:10:28.820Z" }, "lastUpdated": "2026-10-30T16:10:28.820Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T16:10:40.581Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1386073, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 34048, "promptTokensPerSec": null, "peakRssMb": 11972, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 297411, "output": 7049 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1104, "sampleCount": 15, "structuredSampleCount": 0, "firstTokenSampleCount": 15, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-craftbook-api-contract-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 2 }, "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-30T16-11-06-200Z-cd0l", "success": true, "includedInCapability": true, "durationMs": 359259, "timing": { "startedAt": "2026-10-30T16:11:06.200Z", "finishedAt": "2026-10-30T16:17:05.459Z", "timeToFirstArtifactMs": 57922, "timeToLastArtifactWriteMs": 355055, "timeToFirstTokenMs": 18704, "firstTurnTtftMs": 11930, "timeToFirstToolCallMs": 23984 }, "metrics": { "process": { "sampleCount": 72, "peakRssMb": 23891, "peakCpuPercent": 186.4, "firstAtMs": 0, "lastAtMs": 356118 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 72, "peakUsedMb": 65719, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 102318, "totalOutputTokens": 13778, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 102272, "todayTokensCached": 97585, "todayTokensOut": 5225, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 102272, "totalTokensCached": 97585, "totalTokensOut": 5225, "totalCost": 0, "medianOutputTokensPerSec": 70.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.1, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13169, "cachedInputTokens": 13107, "outputTokens": 1662, "cost": 0, "durationMs": 78234, "outputTokensPerSec": 57.79799922882645, "at": "2026-10-30T16:17:01.255Z" }, "lastUpdated": "2026-10-30T16:17:01.255Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T16:17:03.891Z" } }, "source": "engine-log" }, "derived": { "durationMs": 356640, "meanTokensPerSec": 64.77, "genTokensPerSec": 64.77, "genTokens": 13778, "promptTokensPerSec": 1030.95, "peakRssMb": 23891, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 102272, "output": 5225 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-craftbook-api-contract-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 2 }, "trialId": "craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-30T16-17-27-491Z-1jdy", "success": true, "includedInCapability": true, "durationMs": 1864869, "timing": { "startedAt": "2026-10-30T16:17:27.491Z", "finishedAt": "2026-10-30T16:48:32.360Z", "timeToFirstArtifactMs": 346552, "timeToLastArtifactWriteMs": 1851099, "timeToFirstTokenMs": 28447, "firstTurnTtftMs": 21926, "timeToFirstToolCallMs": 41157 }, "metrics": { "process": { "sampleCount": 370, "peakRssMb": 18534, "peakCpuPercent": 112.4, "firstAtMs": 0, "lastAtMs": 1850892 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 370, "peakUsedMb": 71744, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 79998, "totalOutputTokens": 39366, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 79508, "todayTokensCached": 71567, "todayTokensOut": 14139, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 79508, "totalTokensCached": 71567, "totalTokensOut": 14139, "totalCost": 0, "medianOutputTokensPerSec": 25.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.7, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 22451, "cachedInputTokens": 18682, "outputTokens": 1287, "cost": 0, "durationMs": 93925, "outputTokensPerSec": 24.587018195220686, "at": "2026-10-30T16:48:18.590Z" }, "lastUpdated": "2026-10-30T16:48:18.590Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T16:48:20.233Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1851669, "meanTokensPerSec": 23, "genTokensPerSec": 23, "genTokens": 39366, "promptTokensPerSec": null, "peakRssMb": 18534, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 79508, "output": 14139 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 570, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-developer-craftbook-deep-security-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 2 }, "trialId": "craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-30T16-48-53-614Z-mfkk", "success": false, "includedInCapability": true, "durationMs": 265950, "timing": { "startedAt": "2026-10-30T16:48:53.615Z", "finishedAt": "2026-10-30T16:53:19.565Z", "timeToFirstArtifactMs": 51311, "timeToLastArtifactWriteMs": 238752, "timeToFirstTokenMs": 15210, "firstTurnTtftMs": 13571, "timeToFirstToolCallMs": 19955 }, "metrics": { "process": { "sampleCount": 53, "peakRssMb": 12245, "peakCpuPercent": 108.3, "firstAtMs": 0, "lastAtMs": 261007 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 53, "peakUsedMb": 55918, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 64638, "totalOutputTokens": 4826, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 74036, "todayTokensCached": 42788, "todayTokensOut": 2929, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 74036, "totalTokensCached": 42788, "totalTokensOut": 2929, "totalCost": 0, "medianOutputTokensPerSec": 37.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.8, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15345, "cachedInputTokens": 0, "outputTokens": 718, "cost": 0, "durationMs": 35136, "outputTokensPerSec": 34.86566223701197, "at": "2026-10-30T16:52:52.367Z" }, "lastUpdated": "2026-10-30T16:52:52.367Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T16:53:18.244Z" } }, "source": "engine-log" }, "derived": { "durationMs": 263567, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 4826, "promptTokensPerSec": null, "peakRssMb": 12245, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 74036, "output": 2929 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1163.5, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-craftbook-deep-security-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 2 }, "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-30T16-53-41-826Z-7vmd", "success": false, "includedInCapability": true, "durationMs": 318097, "timing": { "startedAt": "2026-10-30T16:53:41.826Z", "finishedAt": "2026-10-30T16:58:59.923Z", "timeToFirstArtifactMs": 40862, "timeToLastArtifactWriteMs": 307345, "timeToFirstTokenMs": 12977, "firstTurnTtftMs": 11106, "timeToFirstToolCallMs": 14618 }, "metrics": { "process": { "sampleCount": 62, "peakRssMb": 28411, "peakCpuPercent": 118.6, "firstAtMs": 0, "lastAtMs": 306018 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 62, "peakUsedMb": 63691, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 76157, "totalOutputTokens": 14588, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 179039, "todayTokensCached": 147501, "todayTokensOut": 7801, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 179039, "totalTokensCached": 147501, "totalTokensOut": 7801, "totalCost": 0, "medianOutputTokensPerSec": 75.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 75.5, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 24127, "cachedInputTokens": 23481, "outputTokens": 55, "cost": 0, "durationMs": 38265, "outputTokensPerSec": 47.604396883146194, "at": "2026-10-30T16:58:49.171Z" }, "lastUpdated": "2026-10-30T16:58:49.171Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T16:58:51.687Z" } }, "source": "engine-log" }, "derived": { "durationMs": 308809, "meanTokensPerSec": 70.73, "genTokensPerSec": 70.73, "genTokens": 14588, "promptTokensPerSec": 957.67, "peakRssMb": 28411, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 179039, "output": 7801 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-craftbook-deep-security-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 2 }, "trialId": "craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-30T16-59-20-292Z-aad5", "success": false, "includedInCapability": true, "durationMs": 518182, "timing": { "startedAt": "2026-10-30T16:59:20.292Z", "finishedAt": "2026-10-30T17:07:58.474Z", "timeToFirstArtifactMs": 340408, "timeToLastArtifactWriteMs": 485700, "timeToFirstTokenMs": 24706, "firstTurnTtftMs": 23057, "timeToFirstToolCallMs": 28570 }, "metrics": { "process": { "sampleCount": 103, "peakRssMb": 18211, "peakCpuPercent": 117.9, "firstAtMs": 0, "lastAtMs": 511664 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 103, "peakUsedMb": 75947, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 27369, "totalOutputTokens": 9960, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 21741, "todayTokensCached": 21596, "todayTokensOut": 532, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 21741, "totalTokensCached": 21596, "totalTokensOut": 532, "totalCost": 0, "medianOutputTokensPerSec": 22.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.8, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 21741, "cachedInputTokens": 21596, "outputTokens": 532, "cost": 0, "durationMs": 484050, "outputTokensPerSec": 22.84427815438367, "at": "2026-10-30T17:07:25.991Z" }, "lastUpdated": "2026-10-30T17:07:25.991Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T17:07:57.071Z" } }, "source": "engine-log" }, "derived": { "durationMs": 515709, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 9960, "promptTokensPerSec": null, "peakRssMb": 18211, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21741, "output": 532 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 502, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-craftbook-deep-security-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 2 }, "trialId": "craftbook-deep-security-review-qwen3-8-27b-q4-2026-10-30T17-08-19-564Z-soq3", "success": true, "includedInCapability": true, "durationMs": 319663, "timing": { "startedAt": "2026-10-30T17:08:19.564Z", "finishedAt": "2026-10-30T17:13:39.227Z", "timeToFirstArtifactMs": 104001, "timeToLastArtifactWriteMs": 311569, "timeToFirstTokenMs": 28589, "firstTurnTtftMs": 26750, "timeToFirstToolCallMs": 32012 }, "metrics": { "process": { "sampleCount": 63, "peakRssMb": 45465, "peakCpuPercent": 105.7, "firstAtMs": 0, "lastAtMs": 311047 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 63, "peakUsedMb": 82432, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 48185, "totalOutputTokens": 4397, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 84823, "todayTokensCached": 69439, "todayTokensOut": 871, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 84823, "totalTokensCached": 69439, "totalTokensOut": 871, "totalCost": 0, "medianOutputTokensPerSec": 33.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.7, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 20748, "cachedInputTokens": 19860, "outputTokens": 83, "cost": 0, "durationMs": 7270, "outputTokensPerSec": 36.734384862566365, "at": "2026-10-30T17:13:31.135Z" }, "lastUpdated": "2026-10-30T17:13:31.135Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T17:13:34.471Z" } }, "source": "engine-log" }, "derived": { "durationMs": 313855, "meanTokensPerSec": 26.64, "genTokensPerSec": 26.64, "genTokens": 4397, "promptTokensPerSec": 394.98, "peakRssMb": 45465, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 84823, "output": 871 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-developer-large-pr-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "large-pr-review", "repetition": 2 }, "trialId": "large-pr-review-gemma4-12b-q4-2026-10-30T17-14-01-149Z-wagn", "success": false, "includedInCapability": true, "durationMs": 1674226, "timing": { "startedAt": "2026-10-30T17:14:01.150Z", "finishedAt": "2026-10-30T17:41:55.376Z", "timeToFirstArtifactMs": 36161, "timeToLastArtifactWriteMs": 1671191, "timeToFirstTokenMs": 17698, "firstTurnTtftMs": 15744, "timeToFirstToolCallMs": 21259 }, "metrics": { "process": { "sampleCount": 334, "peakRssMb": 26060, "peakCpuPercent": 95, "firstAtMs": 0, "lastAtMs": 1670212 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 334, "peakUsedMb": 62036, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 388626, "totalOutputTokens": 57222, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 357618, "todayTokensCached": 182730, "todayTokensOut": 17153, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 357618, "totalTokensCached": 182730, "totalTokensOut": 17153, "totalCost": 0, "medianOutputTokensPerSec": 55, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 55, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7011, "outputTokens": 3192, "cost": 0, "durationMs": 219712, "outputTokensPerSec": 88.07434943458118, "at": "2026-10-30T17:41:52.341Z" }, "lastUpdated": "2026-10-30T17:41:52.341Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T17:41:53.195Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1670996, "meanTokensPerSec": 56.19, "genTokensPerSec": 56.19, "genTokens": 57222, "promptTokensPerSec": 715.4, "peakRssMb": 26060, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 357618, "output": 17153 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-large-pr-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "large-pr-review", "repetition": 2 }, "trialId": "large-pr-review-mlx-qwen3-8-27b-q4-2026-10-30T17-42-15-792Z-imul", "success": true, "includedInCapability": true, "durationMs": 2741769, "timing": { "startedAt": "2026-10-30T17:42:15.792Z", "finishedAt": "2026-10-30T18:27:57.561Z", "timeToFirstArtifactMs": 2146969, "timeToLastArtifactWriteMs": 2734780, "timeToFirstTokenMs": 34806, "firstTurnTtftMs": 33018, "timeToFirstToolCallMs": 37769 }, "metrics": { "process": { "sampleCount": 546, "peakRssMb": 18101, "peakCpuPercent": 120.5, "firstAtMs": 1, "lastAtMs": 2733661 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 546, "peakUsedMb": 76106, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 120002, "totalOutputTokens": 40253, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 228532, "todayTokensCached": 228437, "todayTokensOut": 685, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 228532, "totalTokensCached": 228437, "totalTokensOut": 685, "totalCost": 0, "medianOutputTokensPerSec": 13.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 13.5, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 118232, "cachedInputTokens": 118181, "outputTokens": 188, "cost": 0, "durationMs": 501535, "outputTokensPerSec": 13.79081104092176, "at": "2026-10-30T18:26:24.321Z" }, "lastUpdated": "2026-10-30T18:26:24.321Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T18:27:51.300Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2734437, "meanTokensPerSec": 17, "genTokensPerSec": 17, "genTokens": 40253, "promptTokensPerSec": null, "peakRssMb": 18101, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 228532, "output": 685 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 155, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-large-pr-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "large-pr-review", "repetition": 2 }, "trialId": "large-pr-review-qwen3-8-27b-q4-2026-10-30T18-28-23-562Z-dfyv", "success": true, "includedInCapability": true, "durationMs": 2015917, "timing": { "startedAt": "2026-10-30T18:28:23.563Z", "finishedAt": "2026-10-30T19:01:59.480Z", "timeToFirstArtifactMs": 970453, "timeToLastArtifactWriteMs": 2007483, "timeToFirstTokenMs": 40298, "firstTurnTtftMs": 38215, "timeToFirstToolCallMs": 42466 }, "metrics": { "process": { "sampleCount": 401, "peakRssMb": 41205, "peakCpuPercent": 114.7, "firstAtMs": 0, "lastAtMs": 2006736 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 401, "peakUsedMb": 85179, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 166469, "totalOutputTokens": 29455, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 177405, "todayTokensCached": 177062, "todayTokensOut": 645, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 177405, "totalTokensCached": 177062, "totalTokensOut": 645, "totalCost": 0, "medianOutputTokensPerSec": 17.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 17.8, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 89121, "cachedInputTokens": 88956, "outputTokens": 212, "cost": 0, "durationMs": 572221, "outputTokensPerSec": 17.23820859555903, "at": "2026-10-30T18:54:06.257Z" }, "lastUpdated": "2026-10-30T18:54:06.257Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T19:01:51.968Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2007258, "meanTokensPerSec": 21.98, "genTokensPerSec": 21.98, "genTokens": 29455, "promptTokensPerSec": 253.31, "peakRssMb": 41205, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 177405, "output": 645 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-large-pr-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "large-pr-review", "repetition": 2 }, "trialId": "large-pr-review-mlx-gemma4-12b-q4-2026-10-30T19-02-22-314Z-2sgr", "success": false, "includedInCapability": true, "durationMs": 2398187, "timing": { "startedAt": "2026-10-30T19:02:22.314Z", "finishedAt": "2026-10-30T19:42:20.501Z", "timeToFirstArtifactMs": 483519, "timeToLastArtifactWriteMs": 2394723, "timeToFirstTokenMs": 20674, "firstTurnTtftMs": 18895, "timeToFirstToolCallMs": 21657 }, "metrics": { "process": { "sampleCount": 478, "peakRssMb": 10537, "peakCpuPercent": 101.7, "firstAtMs": 0, "lastAtMs": 2393815 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 478, "peakUsedMb": 66301, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 582109, "totalOutputTokens": 43434, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 808950, "todayTokensCached": 600052, "todayTokensOut": 9181, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 808950, "totalTokensCached": 600052, "totalTokensOut": 9181, "totalCost": 0, "medianOutputTokensPerSec": 37, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 103777, "cachedInputTokens": 103556, "outputTokens": 38, "cost": 0, "durationMs": 101899, "outputTokensPerSec": 33.780505321434646, "at": "2026-10-30T19:42:17.037Z" }, "lastUpdated": "2026-10-30T19:42:17.037Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T19:42:17.977Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2394595, "meanTokensPerSec": 31, "genTokensPerSec": 31, "genTokens": 43434, "promptTokensPerSec": null, "peakRssMb": 10537, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 808950, "output": 9181 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 622, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-developer-craftbook-codemod-sweep", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 2 }, "trialId": "craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-30T19-42-42-401Z-xq97", "success": false, "includedInCapability": true, "durationMs": 2096933, "timing": { "startedAt": "2026-10-30T19:42:42.402Z", "finishedAt": "2026-10-30T20:17:39.335Z", "timeToFirstArtifactMs": 309562, "timeToLastArtifactWriteMs": 1870498, "timeToFirstTokenMs": 28062, "firstTurnTtftMs": 21476, "timeToFirstToolCallMs": 29184 }, "metrics": { "process": { "sampleCount": 418, "peakRssMb": 18320, "peakCpuPercent": 116.5, "firstAtMs": 0, "lastAtMs": 2091895 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 418, "peakUsedMb": 75375, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 135653, "totalOutputTokens": 37825, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 148807, "todayTokensCached": 142411, "todayTokensOut": 13652, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 148807, "totalTokensCached": 142411, "totalTokensOut": 13652, "totalCost": 0, "medianOutputTokensPerSec": 23, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23, "turns": 10 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 19927, "cachedInputTokens": 19423, "outputTokens": 405, "cost": 0, "durationMs": 120716, "outputTokensPerSec": 22.60625510057368, "at": "2026-10-30T20:07:16.684Z" }, "lastUpdated": "2026-10-30T20:07:16.684Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T20:17:36.640Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2093173, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 37825, "promptTokensPerSec": null, "peakRssMb": 18320, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 148807, "output": 13652 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 477, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-craftbook-codemod-sweep", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 2 }, "trialId": "craftbook-codemod-sweep-qwen3-8-27b-q4-2026-10-30T20-18-04-544Z-noxp", "success": false, "includedInCapability": true, "durationMs": 2302746, "timing": { "startedAt": "2026-10-30T20:18:04.544Z", "finishedAt": "2026-10-30T20:56:27.290Z", "timeToFirstArtifactMs": 179618, "timeToLastArtifactWriteMs": 1218491, "timeToFirstTokenMs": 32266, "firstTurnTtftMs": 25486, "timeToFirstToolCallMs": 38287 }, "metrics": { "process": { "sampleCount": 458, "peakRssMb": 48140, "peakCpuPercent": 119.6, "firstAtMs": 0, "lastAtMs": 2292278 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 458, "peakUsedMb": 88639, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 137328, "totalOutputTokens": 41977, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 113113, "todayTokensCached": 105075, "todayTokensOut": 13289, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 113113, "totalTokensCached": 105075, "totalTokensOut": 13289, "totalCost": 0, "medianOutputTokensPerSec": 22.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.9, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 20737, "cachedInputTokens": 19277, "outputTokens": 1653, "cost": 0, "durationMs": 267048, "outputTokensPerSec": 19.164828335806586, "at": "2026-10-30T20:44:01.479Z" }, "lastUpdated": "2026-10-30T20:44:01.479Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T20:56:18.647Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2293055, "meanTokensPerSec": 22.25, "genTokensPerSec": 22.25, "genTokens": 41977, "promptTokensPerSec": 365.48, "peakRssMb": 48140, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 113113, "output": 13289 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-craftbook-codemod-sweep", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 2 }, "trialId": "craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-30T20-56-48-802Z-w08m", "success": false, "includedInCapability": true, "durationMs": 1249050, "timing": { "startedAt": "2026-10-30T20:56:48.802Z", "finishedAt": "2026-10-30T21:17:37.852Z", "timeToFirstArtifactMs": 42999, "timeToLastArtifactWriteMs": 1104706, "timeToFirstTokenMs": 20349, "firstTurnTtftMs": 13827, "timeToFirstToolCallMs": 28950 }, "metrics": { "process": { "sampleCount": 249, "peakRssMb": 11198, "peakCpuPercent": 138.3, "firstAtMs": 0, "lastAtMs": 1245039 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 249, "peakUsedMb": 59569, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 123433, "totalOutputTokens": 30754, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 87189, "todayTokensCached": 85617, "todayTokensOut": 3414, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 87189, "totalTokensCached": 85617, "totalTokensOut": 3414, "totalCost": 0, "medianOutputTokensPerSec": 37.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14793, "cachedInputTokens": 14639, "outputTokens": 790, "cost": 0, "durationMs": 346184, "outputTokensPerSec": 34.670257092126455, "at": "2026-10-30T21:07:52.843Z" }, "lastUpdated": "2026-10-30T21:07:52.843Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T21:17:35.955Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1246085, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 30754, "promptTokensPerSec": null, "peakRssMb": 11198, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 87189, "output": 3414 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1036, "sampleCount": 14, "structuredSampleCount": 0, "firstTokenSampleCount": 14, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-craftbook-codemod-sweep", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 2 }, "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-30T21-17-58-757Z-oyet", "success": false, "includedInCapability": true, "durationMs": 1405117, "timing": { "startedAt": "2026-10-30T21:17:58.757Z", "finishedAt": "2026-10-30T21:41:23.874Z", "timeToFirstArtifactMs": 28407, "timeToLastArtifactWriteMs": 1393225, "timeToFirstTokenMs": 18345, "firstTurnTtftMs": 11658, "timeToFirstToolCallMs": 22340 }, "metrics": { "process": { "sampleCount": 278, "peakRssMb": 27799, "peakCpuPercent": 114.7, "firstAtMs": 0, "lastAtMs": 1389562 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 278, "peakUsedMb": 69586, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 246674, "totalOutputTokens": 30047, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 141331, "todayTokensCached": 126706, "todayTokensOut": 4923, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 141331, "totalTokensCached": 126706, "totalTokensOut": 4923, "totalCost": 0, "medianOutputTokensPerSec": 67.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 67.7, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16509, "cachedInputTokens": 16458, "outputTokens": 243, "cost": 0, "durationMs": 41588, "outputTokensPerSec": 65.4472385727634, "at": "2026-10-30T21:23:04.404Z" }, "lastUpdated": "2026-10-30T21:23:04.404Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T21:41:13.674Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1393867, "meanTokensPerSec": 38.57, "genTokensPerSec": 38.57, "genTokens": 30047, "promptTokensPerSec": 498.48, "peakRssMb": 27799, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 141331, "output": 4923 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-developer-craftbook-refactor-module", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 2 }, "trialId": "craftbook-refactor-module-qwen3-8-27b-q4-2026-10-30T21-41-50-785Z-psqe", "success": false, "includedInCapability": true, "durationMs": 2600353, "timing": { "startedAt": "2026-10-30T21:41:50.786Z", "finishedAt": "2026-10-30T22:25:11.139Z", "timeToFirstArtifactMs": 50909, "timeToLastArtifactWriteMs": 2575311, "timeToFirstTokenMs": 24997, "firstTurnTtftMs": 18274, "timeToFirstToolCallMs": 50908 }, "metrics": { "process": { "sampleCount": 518, "peakRssMb": 51558, "peakCpuPercent": 141.7, "firstAtMs": 0, "lastAtMs": 2594547 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 518, "peakUsedMb": 92511, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 245743, "totalOutputTokens": 37062, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 19, "todayTokensIn": 404054, "todayTokensCached": 290467, "todayTokensOut": 17291, "todayCost": 0, "totalTurns": 19, "totalTokensIn": 404054, "totalTokensCached": 290467, "totalTokensOut": 17291, "totalCost": 0, "medianOutputTokensPerSec": 29.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.5, "turns": 19 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 42445, "cachedInputTokens": 39802, "outputTokens": 765, "cost": 0, "durationMs": 25720, "outputTokensPerSec": 40.68614737864585, "at": "2026-10-30T22:24:46.097Z" }, "lastUpdated": "2026-10-30T22:24:46.097Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T22:25:07.177Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2595344, "meanTokensPerSec": 26.74, "genTokensPerSec": 26.74, "genTokens": 37062, "promptTokensPerSec": 360.56, "peakRssMb": 51558, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 404054, "output": 17291 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-developer-craftbook-refactor-module", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 2 }, "trialId": "craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-30T22-25-32-724Z-5c3y", "success": false, "includedInCapability": true, "durationMs": 1758140, "timing": { "startedAt": "2026-10-30T22:25:32.725Z", "finishedAt": "2026-10-30T22:54:50.865Z", "timeToFirstArtifactMs": 24408, "timeToLastArtifactWriteMs": 1755422, "timeToFirstTokenMs": 17353, "firstTurnTtftMs": 10835, "timeToFirstToolCallMs": 24407 }, "metrics": { "process": { "sampleCount": 350, "peakRssMb": 10912, "peakCpuPercent": 121.4, "firstAtMs": 0, "lastAtMs": 1753043 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 350, "peakUsedMb": 60029, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 552124, "totalOutputTokens": 10834, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 32, "todayTokensIn": 900697, "todayTokensCached": 585862, "todayTokensOut": 8009, "todayCost": 0, "totalTurns": 32, "totalTokensIn": 900697, "totalTokensCached": 585862, "totalTokensOut": 8009, "totalCost": 0, "medianOutputTokensPerSec": 48.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 48.6, "turns": 32 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 46871, "cachedInputTokens": 1655, "outputTokens": 281, "cost": 0, "durationMs": 45555, "outputTokensPerSec": 41.89667573175329, "at": "2026-10-30T22:54:48.147Z" }, "lastUpdated": "2026-10-30T22:54:48.147Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T22:54:49.267Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1755588, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 10834, "promptTokensPerSec": null, "peakRssMb": 10912, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 900697, "output": 8009 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1105.5, "sampleCount": 36, "structuredSampleCount": 0, "firstTokenSampleCount": 36, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-craftbook-refactor-module", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 2 }, "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-30T22-55-14-115Z-a6e2", "success": false, "includedInCapability": true, "durationMs": 1609638, "timing": { "startedAt": "2026-10-30T22:55:14.116Z", "finishedAt": "2026-10-30T23:22:03.754Z", "timeToFirstArtifactMs": 19784, "timeToLastArtifactWriteMs": 1598909, "timeToFirstTokenMs": 15999, "firstTurnTtftMs": 9279, "timeToFirstToolCallMs": 19786 }, "metrics": { "process": { "sampleCount": 320, "peakRssMb": 27416, "peakCpuPercent": 116.9, "firstAtMs": 0, "lastAtMs": 1600358 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 320, "peakUsedMb": 66688, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 781229, "totalOutputTokens": 28023, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 107481, "todayTokensCached": 64782, "todayTokensOut": 4774, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 107481, "totalTokensCached": 64782, "totalTokensOut": 4774, "totalCost": 0, "medianOutputTokensPerSec": 87.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 87.5, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16750, "outputTokens": 102, "cost": 0, "durationMs": 48333, "outputTokensPerSec": 115.32800540785578, "at": "2026-10-30T23:00:07.585Z" }, "lastUpdated": "2026-10-30T23:00:07.585Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-30T23:21:56.051Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1600884, "meanTokensPerSec": 66.56, "genTokensPerSec": 66.56, "genTokens": 28023, "promptTokensPerSec": 778.45, "peakRssMb": 27416, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 107481, "output": 4774 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-craftbook-refactor-module", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 2 }, "trialId": "craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-30T23-22-29-040Z-ku0t", "success": false, "includedInCapability": true, "durationMs": 2260956, "timing": { "startedAt": "2026-10-30T23:22:29.040Z", "finishedAt": "2026-10-31T00:00:09.996Z", "timeToFirstArtifactMs": 39149, "timeToLastArtifactWriteMs": 2254241, "timeToFirstTokenMs": 22523, "firstTurnTtftMs": 16008, "timeToFirstToolCallMs": 39148 }, "metrics": { "process": { "sampleCount": 450, "peakRssMb": 19153, "peakCpuPercent": 111.4, "firstAtMs": 0, "lastAtMs": 2253615 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 450, "peakUsedMb": 75523, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 229544, "totalOutputTokens": 104199, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 22651, "todayTokensCached": 2267, "todayTokensOut": 161, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 356844, "totalTokensCached": 235171, "totalTokensOut": 13859, "totalCost": 0, "medianOutputTokensPerSec": 23.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.3, "turns": 23 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 22651, "cachedInputTokens": 2267, "outputTokens": 161, "cost": 0, "durationMs": 98929, "outputTokensPerSec": 13.427948710680681, "at": "2026-10-31T00:00:03.286Z" }, "lastUpdated": "2026-10-31T00:00:03.286Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T00:00:07.277Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2257165, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 104199, "promptTokensPerSec": null, "peakRssMb": 19153, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 356844, "output": 13859 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 463, "sampleCount": 24, "structuredSampleCount": 0, "firstTokenSampleCount": 24, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-developer-craftbook-bug-fix-tdd", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 2 }, "trialId": "craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-31T00-00-32-268Z-0xx7", "success": false, "includedInCapability": true, "durationMs": 2064049, "timing": { "startedAt": "2026-10-31T00:00:32.268Z", "finishedAt": "2026-10-31T00:34:56.317Z", "timeToFirstArtifactMs": 31075, "timeToLastArtifactWriteMs": 1158370, "timeToFirstTokenMs": 21591, "firstTurnTtftMs": 15062, "timeToFirstToolCallMs": 22306 }, "metrics": { "process": { "sampleCount": 411, "peakRssMb": 11375, "peakCpuPercent": 144.7, "firstAtMs": 0, "lastAtMs": 2059303 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 411, "peakUsedMb": 60320, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 420649, "totalOutputTokens": 38676, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 380116, "todayTokensCached": 184545, "todayTokensOut": 5543, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 380116, "totalTokensCached": 184545, "totalTokensOut": 5543, "totalCost": 0, "medianOutputTokensPerSec": 38.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.8, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 33293, "cachedInputTokens": 1613, "outputTokens": 453, "cost": 0, "durationMs": 32537, "outputTokensPerSec": 45.74828056254729, "at": "2026-10-31T00:19:50.638Z" }, "lastUpdated": "2026-10-31T00:19:50.638Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T00:34:54.174Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2060846, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 38676, "promptTokensPerSec": null, "peakRssMb": 11375, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 380116, "output": 5543 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1069, "sampleCount": 23, "structuredSampleCount": 0, "firstTokenSampleCount": 23, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-developer-craftbook-bug-fix-tdd", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 2 }, "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-31T00-35-18-617Z-dm02", "success": false, "includedInCapability": true, "durationMs": 1568910, "timing": { "startedAt": "2026-10-31T00:35:18.618Z", "finishedAt": "2026-10-31T01:01:27.528Z", "timeToFirstArtifactMs": 28041, "timeToLastArtifactWriteMs": 1560566, "timeToFirstTokenMs": 19908, "firstTurnTtftMs": 13178, "timeToFirstToolCallMs": 23248 }, "metrics": { "process": { "sampleCount": 311, "peakRssMb": 30306, "peakCpuPercent": 104.8, "firstAtMs": 0, "lastAtMs": 1555221 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 311, "peakUsedMb": 69905, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 702832, "totalOutputTokens": 30973, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 221374, "todayTokensCached": 174163, "todayTokensOut": 3793, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 221374, "totalTokensCached": 174163, "totalTokensOut": 3793, "totalCost": 0, "medianOutputTokensPerSec": 88.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88.4, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 20649, "cachedInputTokens": 20202, "outputTokens": 197, "cost": 0, "durationMs": 101267, "outputTokensPerSec": 66.77655270816399, "at": "2026-10-31T00:44:12.615Z" }, "lastUpdated": "2026-10-31T00:44:12.615Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T01:01:19.914Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1560248, "meanTokensPerSec": 60.61, "genTokensPerSec": 60.61, "genTokens": 30973, "promptTokensPerSec": 788.27, "peakRssMb": 30306, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 221374, "output": 3793 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-developer-craftbook-bug-fix-tdd", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 2 }, "trialId": "craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-31T01-01-49-395Z-ikz8", "success": false, "includedInCapability": true, "durationMs": 2881574, "timing": { "startedAt": "2026-10-31T01:01:49.396Z", "finishedAt": "2026-10-31T01:49:50.970Z", "timeToFirstArtifactMs": 140277, "timeToLastArtifactWriteMs": 2873733, "timeToFirstTokenMs": 31025, "firstTurnTtftMs": 24478, "timeToFirstToolCallMs": 39320 }, "metrics": { "process": { "sampleCount": 574, "peakRssMb": 18336, "peakCpuPercent": 120.3, "firstAtMs": 0, "lastAtMs": 2874971 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 574, "peakUsedMb": 76289, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 478622, "totalOutputTokens": 36034, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 448152, "todayTokensCached": 190433, "todayTokensOut": 15442, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 448152, "totalTokensCached": 190433, "totalTokensOut": 15442, "totalCost": 0, "medianOutputTokensPerSec": 24.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.2, "turns": 22 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 29784, "cachedInputTokens": 2255, "outputTokens": 219, "cost": 0, "durationMs": 151672, "outputTokensPerSec": 19.998392483776087, "at": "2026-10-31T01:49:43.131Z" }, "lastUpdated": "2026-10-31T01:49:43.131Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T01:49:48.246Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2877777, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 36034, "promptTokensPerSec": null, "peakRssMb": 18336, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 448152, "output": 15442 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 477, "sampleCount": 23, "structuredSampleCount": 0, "firstTokenSampleCount": 23, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-developer-craftbook-bug-fix-tdd", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 2 }, "trialId": "craftbook-bug-fix-tdd-qwen3-8-27b-q4-2026-10-31T01-50-38-341Z-3l4l", "success": false, "includedInCapability": true, "durationMs": 7981093, "timing": { "startedAt": "2026-10-31T01:50:38.341Z", "finishedAt": "2026-10-31T04:03:39.434Z", "timeToFirstArtifactMs": 62650, "timeToLastArtifactWriteMs": 7067566, "timeToFirstTokenMs": 33050, "firstTurnTtftMs": 26216, "timeToFirstToolCallMs": 37226 }, "metrics": { "process": { "sampleCount": 1589, "peakRssMb": 50847, "peakCpuPercent": 109.2, "firstAtMs": 0, "lastAtMs": 7965283 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1589, "peakUsedMb": 92295, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 937741, "totalOutputTokens": 60953, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 25, "todayTokensIn": 967043, "todayTokensCached": 724558, "todayTokensOut": 14043, "todayCost": 0, "totalTurns": 25, "totalTokensIn": 967043, "totalTokensCached": 724558, "totalTokensOut": 14043, "totalCost": 0, "medianOutputTokensPerSec": 30.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.6, "turns": 25 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 56706, "outputTokens": 574, "cost": 0, "durationMs": 208659, "outputTokensPerSec": 23.886617853758622, "at": "2026-10-31T03:48:25.907Z" }, "lastUpdated": "2026-10-31T03:48:25.907Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T04:03:29.308Z" } }, "source": "engine-log" }, "derived": { "durationMs": 7969827, "meanTokensPerSec": 19.42, "genTokensPerSec": 19.42, "genTokens": 60953, "promptTokensPerSec": 317.9, "peakRssMb": 50847, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 967043, "output": 14043 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-find-vs-create", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 2 }, "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-31T04-04-19-398Z-4zfy", "success": false, "includedInCapability": true, "durationMs": 244912, "timing": { "startedAt": "2026-10-31T04:04:19.398Z", "finishedAt": "2026-10-31T04:08:24.310Z", "timeToFirstArtifactMs": 74542, "timeToLastArtifactWriteMs": 234609, "timeToFirstTokenMs": 13281, "firstTurnTtftMs": 11447, "timeToFirstToolCallMs": 23791 }, "metrics": { "process": { "sampleCount": 48, "peakRssMb": 24867, "peakCpuPercent": 123.7, "firstAtMs": 0, "lastAtMs": 235792 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 48, "peakUsedMb": 68125, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 106008, "totalOutputTokens": 6887, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 77563, "todayTokensCached": 17648, "todayTokensOut": 2692, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 77563, "totalTokensCached": 17648, "totalTokensOut": 2692, "totalCost": 0, "medianOutputTokensPerSec": 71.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 71.4, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15097, "outputTokens": 145, "cost": 0, "durationMs": 32167, "outputTokensPerSec": 86.62487818376506, "at": "2026-10-31T04:08:14.007Z" }, "lastUpdated": "2026-10-31T04:08:14.007Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T04:08:17.528Z" } }, "source": "engine-log" }, "derived": { "durationMs": 237088, "meanTokensPerSec": 62.17, "genTokensPerSec": 62.17, "genTokens": 6887, "promptTokensPerSec": 1001.22, "peakRssMb": 24867, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 77563, "output": 2692 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-find-vs-create", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 2 }, "trialId": "craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-31T04-09-10-016Z-b5qq", "success": true, "includedInCapability": true, "durationMs": 81304, "timing": { "startedAt": "2026-10-31T04:09:10.016Z", "finishedAt": "2026-10-31T04:10:31.320Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 27521, "firstTurnTtftMs": 25832, "timeToFirstToolCallMs": 49446 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 17981, "peakCpuPercent": 109.2, "firstAtMs": 0, "lastAtMs": 75242 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 77079, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13251, "totalOutputTokens": 1149, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T04:10:27.095Z" } }, "source": "engine-log" }, "derived": { "durationMs": 76016, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 1149, "promptTokensPerSec": null, "peakRssMb": 17981, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 572, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-find-vs-create", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 2 }, "trialId": "craftbook-find-vs-create-qwen3-8-27b-q4-2026-10-31T04-10-52-772Z-gjm3", "success": true, "includedInCapability": true, "durationMs": 82913, "timing": { "startedAt": "2026-10-31T04:10:52.772Z", "finishedAt": "2026-10-31T04:12:15.685Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 33023, "firstTurnTtftMs": 31135, "timeToFirstToolCallMs": 47334 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 37606, "peakCpuPercent": 114.1, "firstAtMs": 0, "lastAtMs": 75246 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 80141, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13352, "totalOutputTokens": 1079, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T04:12:09.845Z" } }, "source": "engine-log" }, "derived": { "durationMs": 76019, "meanTokensPerSec": 29.45, "genTokensPerSec": 29.45, "genTokens": 1079, "promptTokensPerSec": 455.05, "peakRssMb": 37606, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-find-vs-create", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 2 }, "trialId": "craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-31T04-12-55-808Z-l1jn", "success": false, "includedInCapability": true, "durationMs": 524626, "timing": { "startedAt": "2026-10-31T04:12:55.809Z", "finishedAt": "2026-10-31T04:21:40.435Z", "timeToFirstArtifactMs": 337880, "timeToLastArtifactWriteMs": 515515, "timeToFirstTokenMs": 15063, "firstTurnTtftMs": 13382, "timeToFirstToolCallMs": 45041 }, "metrics": { "process": { "sampleCount": 104, "peakRssMb": 11211, "peakCpuPercent": 119.8, "firstAtMs": 0, "lastAtMs": 516895 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 104, "peakUsedMb": 60268, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 65551, "totalOutputTokens": 14452, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 92737, "todayTokensCached": 75698, "todayTokensOut": 13654, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 92737, "totalTokensCached": 75698, "totalTokensOut": 13654, "totalCost": 0, "medianOutputTokensPerSec": 35.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.3, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11534, "cachedInputTokens": 11317, "outputTokens": 214, "cost": 0, "durationMs": 15463, "outputTokensPerSec": 35.53370594163961, "at": "2026-10-31T04:21:31.324Z" }, "lastUpdated": "2026-10-31T04:21:31.324Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T04:21:35.813Z" } }, "source": "engine-log" }, "derived": { "durationMs": 518950, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 14452, "promptTokensPerSec": null, "peakRssMb": 11211, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 92737, "output": 13654 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1011, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-edit-midtask", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 2 }, "trialId": "craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-31T04-22-03-008Z-t82e", "success": true, "includedInCapability": true, "durationMs": 2467143, "timing": { "startedAt": "2026-10-31T04:22:03.008Z", "finishedAt": "2026-10-31T05:03:10.151Z", "timeToFirstArtifactMs": 2164215, "timeToLastArtifactWriteMs": 2423743, "timeToFirstTokenMs": 39989, "firstTurnTtftMs": 38312, "timeToFirstToolCallMs": 47477 }, "metrics": { "process": { "sampleCount": 492, "peakRssMb": 18265, "peakCpuPercent": 112.1, "firstAtMs": 0, "lastAtMs": 2462944 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 492, "peakUsedMb": 76493, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 51464, "totalOutputTokens": 51954, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 136954, "todayTokensCached": 127379, "todayTokensOut": 9608, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 136954, "totalTokensCached": 127379, "totalTokensOut": 9608, "totalCost": 0, "medianOutputTokensPerSec": 23.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.3, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 27182, "cachedInputTokens": 26872, "outputTokens": 161, "cost": 0, "durationMs": 11485, "outputTokensPerSec": 26.28407134424659, "at": "2026-10-31T05:03:06.400Z" }, "lastUpdated": "2026-10-31T05:03:06.400Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T05:03:07.565Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2463489, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 51954, "promptTokensPerSec": null, "peakRssMb": 18265, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 136954, "output": 9608 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 305, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-edit-midtask", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 2 }, "trialId": "craftbook-edit-midtask-qwen3-8-27b-q4-2026-10-31T05-03-32-619Z-181v", "success": true, "includedInCapability": true, "durationMs": 773516, "timing": { "startedAt": "2026-10-31T05:03:32.619Z", "finishedAt": "2026-10-31T05:16:26.135Z", "timeToFirstArtifactMs": 591781, "timeToLastArtifactWriteMs": 767226, "timeToFirstTokenMs": 49077, "firstTurnTtftMs": 47220, "timeToFirstToolCallMs": 56378 }, "metrics": { "process": { "sampleCount": 154, "peakRssMb": 46123, "peakCpuPercent": 105.9, "firstAtMs": 0, "lastAtMs": 767444 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 154, "peakUsedMb": 90930, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44375, "totalOutputTokens": 13975, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 109175, "todayTokensCached": 106554, "todayTokensOut": 1648, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 109175, "totalTokensCached": 106554, "totalTokensOut": 1648, "totalCost": 0, "medianOutputTokensPerSec": 23.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.2, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7591, "cachedInputTokens": 7301, "outputTokens": 59, "cost": 0, "durationMs": 80391, "outputTokensPerSec": 39.56300613363019, "at": "2026-10-31T05:16:19.844Z" }, "lastUpdated": "2026-10-31T05:16:19.844Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T05:16:21.655Z" } }, "source": "engine-log" }, "derived": { "durationMs": 767985, "meanTokensPerSec": 22.51, "genTokensPerSec": 22.51, "genTokens": 13975, "promptTokensPerSec": 360.08, "peakRssMb": 46123, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 109175, "output": 1648 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-edit-midtask", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 2 }, "trialId": "craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-31T05-16-47-886Z-ymnx", "success": true, "includedInCapability": true, "durationMs": 646560, "timing": { "startedAt": "2026-10-31T05:16:47.886Z", "finishedAt": "2026-10-31T05:27:34.446Z", "timeToFirstArtifactMs": 112751, "timeToLastArtifactWriteMs": 639873, "timeToFirstTokenMs": 21004, "firstTurnTtftMs": 19433, "timeToFirstToolCallMs": 32026 }, "metrics": { "process": { "sampleCount": 129, "peakRssMb": 11284, "peakCpuPercent": 119.2, "firstAtMs": 0, "lastAtMs": 642396 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 129, "peakUsedMb": 60253, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 81386, "totalOutputTokens": 14739, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 193024, "todayTokensCached": 169306, "todayTokensOut": 3513, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 193024, "totalTokensCached": 169306, "totalTokensOut": 3513, "totalCost": 0, "medianOutputTokensPerSec": 36, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9555, "cachedInputTokens": 9223, "outputTokens": 30, "cost": 0, "durationMs": 113056, "outputTokensPerSec": 38.13175230182864, "at": "2026-10-31T05:27:27.759Z" }, "lastUpdated": "2026-10-31T05:27:27.759Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T05:27:31.769Z" } }, "source": "engine-log" }, "derived": { "durationMs": 642924, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 14739, "promptTokensPerSec": null, "peakRssMb": 11284, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 193024, "output": 3513 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 962, "sampleCount": 15, "structuredSampleCount": 0, "firstTokenSampleCount": 15, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-edit-midtask", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 2 }, "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-31T05-27-56-489Z-y68f", "success": false, "includedInCapability": true, "durationMs": 2692826, "timing": { "startedAt": "2026-10-31T05:27:56.490Z", "finishedAt": "2026-10-31T06:12:49.316Z", "timeToFirstArtifactMs": 367746, "timeToLastArtifactWriteMs": 2352178, "timeToFirstTokenMs": 19243, "firstTurnTtftMs": 17349, "timeToFirstToolCallMs": 27316 }, "metrics": { "process": { "sampleCount": 537, "peakRssMb": 29316, "peakCpuPercent": 104.8, "firstAtMs": 0, "lastAtMs": 2691059 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 537, "peakUsedMb": 82094, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 436174, "totalOutputTokens": 61615, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 49, "todayTokensIn": 953111, "todayTokensCached": 811109, "todayTokensOut": 23436, "todayCost": 0, "totalTurns": 49, "totalTokensIn": 953111, "totalTokensCached": 811109, "totalTokensOut": 23436, "totalCost": 0, "medianOutputTokensPerSec": 44.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 44.5, "turns": 49 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22438, "outputTokens": 589, "cost": 0, "durationMs": 36441, "outputTokensPerSec": 52.13324726027817, "at": "2026-10-31T06:07:45.733Z" }, "lastUpdated": "2026-10-31T06:07:45.733Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T06:12:49.146Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2691608, "meanTokensPerSec": 48.9, "genTokensPerSec": 48.9, "genTokens": 61615, "promptTokensPerSec": 847.95, "peakRssMb": 29316, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 953111, "output": 23436 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-invoice-run", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 2 }, "trialId": "craftbook-invoice-run-qwen3-8-27b-q4-2026-10-31T06-13-10-708Z-tql8", "success": false, "includedInCapability": true, "durationMs": 1594050, "timing": { "startedAt": "2026-10-31T06:13:10.708Z", "finishedAt": "2026-10-31T06:39:44.758Z", "timeToFirstArtifactMs": 67087, "timeToLastArtifactWriteMs": 1584842, "timeToFirstTokenMs": 25393, "firstTurnTtftMs": 18820, "timeToFirstToolCallMs": 26540 }, "metrics": { "process": { "sampleCount": 317, "peakRssMb": 52938, "peakCpuPercent": 147.3, "firstAtMs": 0, "lastAtMs": 1585133 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 317, "peakUsedMb": 98850, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 273798, "totalOutputTokens": 26145, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 245454, "todayTokensCached": 219833, "todayTokensOut": 3225, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 245454, "totalTokensCached": 219833, "totalTokensOut": 3225, "totalCost": 0, "medianOutputTokensPerSec": 33.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.5, "turns": 15 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 23973, "cachedInputTokens": 22257, "outputTokens": 815, "cost": 0, "durationMs": 177822, "outputTokensPerSec": 33.460721121290966, "at": "2026-10-31T06:39:35.550Z" }, "lastUpdated": "2026-10-31T06:39:35.550Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T06:39:38.353Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1586698, "meanTokensPerSec": 34.01, "genTokensPerSec": 34.01, "genTokens": 26145, "promptTokensPerSec": 380.8, "peakRssMb": 52938, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 245454, "output": 3225 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-invoice-run", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 2 }, "trialId": "craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-31T06-40-09-922Z-nrko", "success": false, "includedInCapability": true, "durationMs": 909591, "timing": { "startedAt": "2026-10-31T06:40:09.922Z", "finishedAt": "2026-10-31T06:55:19.513Z", "timeToFirstArtifactMs": 43727, "timeToLastArtifactWriteMs": 896125, "timeToFirstTokenMs": 20437, "firstTurnTtftMs": 13907, "timeToFirstToolCallMs": 30385 }, "metrics": { "process": { "sampleCount": 180, "peakRssMb": 12544, "peakCpuPercent": 143.4, "firstAtMs": 0, "lastAtMs": 898238 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 180, "peakUsedMb": 59829, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 176143, "totalOutputTokens": 21726, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 141128, "todayTokensCached": 94904, "todayTokensOut": 11872, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 141128, "totalTokensCached": 94904, "totalTokensOut": 11872, "totalCost": 0, "medianOutputTokensPerSec": 35.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.1, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19524, "cachedInputTokens": 19160, "outputTokens": 394, "cost": 0, "durationMs": 61329, "outputTokensPerSec": 33.98148500408851, "at": "2026-10-31T06:55:06.052Z" }, "lastUpdated": "2026-10-31T06:55:06.052Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T06:55:10.508Z" } }, "source": "engine-log" }, "derived": { "durationMs": 899521, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 21726, "promptTokensPerSec": null, "peakRssMb": 12544, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 141128, "output": 11872 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1058, "sampleCount": 12, "structuredSampleCount": 0, "firstTokenSampleCount": 12, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-invoice-run", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 2 }, "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-31T06-55-41-908Z-j2ot", "success": true, "includedInCapability": true, "durationMs": 248054, "timing": { "startedAt": "2026-10-31T06:55:41.908Z", "finishedAt": "2026-10-31T06:59:49.962Z", "timeToFirstArtifactMs": 28143, "timeToLastArtifactWriteMs": 238245, "timeToFirstTokenMs": 18802, "firstTurnTtftMs": 12068, "timeToFirstToolCallMs": 22413 }, "metrics": { "process": { "sampleCount": 49, "peakRssMb": 28760, "peakCpuPercent": 141.6, "firstAtMs": 0, "lastAtMs": 240766 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 83499, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 91172, "totalOutputTokens": 7987, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 76063, "todayTokensCached": 63539, "todayTokensOut": 478, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 76063, "totalTokensCached": 63539, "totalTokensOut": 478, "totalCost": 0, "medianOutputTokensPerSec": 96.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 96.3, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16541, "cachedInputTokens": 9224, "outputTokens": 79, "cost": 0, "durationMs": 46699, "outputTokensPerSec": 70.34695478855329, "at": "2026-10-31T06:59:40.153Z" }, "lastUpdated": "2026-10-31T06:59:40.153Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T06:59:44.242Z" } }, "source": "engine-log" }, "derived": { "durationMs": 241284, "meanTokensPerSec": 84.57, "genTokensPerSec": 84.57, "genTokens": 7987, "promptTokensPerSec": 986.82, "peakRssMb": 28760, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 76063, "output": 478 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-invoice-run", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 2 }, "trialId": "craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-31T07-00-15-530Z-uj35", "success": false, "includedInCapability": true, "durationMs": 928423, "timing": { "startedAt": "2026-10-31T07:00:15.530Z", "finishedAt": "2026-10-31T07:15:43.953Z", "timeToFirstArtifactMs": 86819, "timeToLastArtifactWriteMs": 913123, "timeToFirstTokenMs": 27624, "firstTurnTtftMs": 21112, "timeToFirstToolCallMs": 28780 }, "metrics": { "process": { "sampleCount": 183, "peakRssMb": 20399, "peakCpuPercent": 140.5, "firstAtMs": 1, "lastAtMs": 912955 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 183, "peakUsedMb": 76298, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 179554, "totalOutputTokens": 12181, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 127499, "todayTokensCached": 79506, "todayTokensOut": 2085, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 127499, "totalTokensCached": 79506, "totalTokensOut": 2085, "totalCost": 0, "medianOutputTokensPerSec": 30.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.3, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 17783, "cachedInputTokens": 1459, "outputTokens": 251, "cost": 0, "durationMs": 83746, "outputTokensPerSec": 14.464430870441088, "at": "2026-10-31T07:15:28.658Z" }, "lastUpdated": "2026-10-31T07:15:28.658Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T07:15:30.834Z" } }, "source": "engine-log" }, "derived": { "durationMs": 914242, "meanTokensPerSec": 27, "genTokensPerSec": 27, "genTokens": 12181, "promptTokensPerSec": null, "peakRssMb": 20399, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 127499, "output": 2085 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 532, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-author-linear", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 2 }, "trialId": "craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-31T07-16-05-763Z-q5fx", "success": false, "includedInCapability": true, "durationMs": 1721207, "timing": { "startedAt": "2026-10-31T07:16:05.763Z", "finishedAt": "2026-10-31T07:44:46.970Z", "timeToFirstArtifactMs": 160639, "timeToLastArtifactWriteMs": 1082360, "timeToFirstTokenMs": 20920, "firstTurnTtftMs": 19249, "timeToFirstToolCallMs": 46024 }, "metrics": { "process": { "sampleCount": 343, "peakRssMb": 13066, "peakCpuPercent": 117, "firstAtMs": 0, "lastAtMs": 1717955 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 343, "peakUsedMb": 61078, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 62299, "totalOutputTokens": 31891, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 313248, "todayTokensCached": 311106, "todayTokensOut": 2950, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 313248, "totalTokensCached": 311106, "totalTokensOut": 2950, "totalCost": 0, "medianOutputTokensPerSec": 37.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.4, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10351, "cachedInputTokens": 10206, "outputTokens": 33, "cost": 0, "durationMs": 2565, "outputTokensPerSec": 49.855872363258854, "at": "2026-10-31T07:35:19.243Z" }, "lastUpdated": "2026-10-31T07:35:19.243Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T07:44:45.579Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1718751, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 31891, "promptTokensPerSec": null, "peakRssMb": 13066, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 313248, "output": 2950 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 577.5, "sampleCount": 16, "structuredSampleCount": 0, "firstTokenSampleCount": 16, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-author-linear", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 2 }, "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-31T07-45-09-615Z-g1y0", "success": true, "includedInCapability": true, "durationMs": 440630, "timing": { "startedAt": "2026-10-31T07:45:09.616Z", "finishedAt": "2026-10-31T07:52:30.246Z", "timeToFirstArtifactMs": 151825, "timeToLastArtifactWriteMs": 439257, "timeToFirstTokenMs": 15095, "firstTurnTtftMs": 13208, "timeToFirstToolCallMs": 25333 }, "metrics": { "process": { "sampleCount": 88, "peakRssMb": 29658, "peakCpuPercent": 112.1, "firstAtMs": 0, "lastAtMs": 436460 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 88, "peakUsedMb": 74808, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 101554, "totalOutputTokens": 15887, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 267430, "todayTokensCached": 256531, "todayTokensOut": 3670, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 267430, "totalTokensCached": 256531, "totalTokensOut": 3670, "totalCost": 0, "medianOutputTokensPerSec": 46.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 46.7, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 28869, "cachedInputTokens": 28644, "outputTokens": 47, "cost": 0, "durationMs": 11519, "outputTokensPerSec": 36.96673476046359, "at": "2026-10-31T07:51:35.066Z" }, "lastUpdated": "2026-10-31T07:51:35.066Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T07:52:29.412Z" } }, "source": "engine-log" }, "derived": { "durationMs": 438742, "meanTokensPerSec": 57.16, "genTokensPerSec": 57.16, "genTokens": 15887, "promptTokensPerSec": 851.66, "peakRssMb": 29658, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 267430, "output": 3670 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-author-linear", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 2 }, "trialId": "craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-31T07-52-52-532Z-liak", "success": true, "includedInCapability": true, "durationMs": 1955691, "timing": { "startedAt": "2026-10-31T07:52:52.532Z", "finishedAt": "2026-10-31T08:25:28.223Z", "timeToFirstArtifactMs": 719945, "timeToLastArtifactWriteMs": 1943200, "timeToFirstTokenMs": 42318, "firstTurnTtftMs": 40751, "timeToFirstToolCallMs": 50403 }, "metrics": { "process": { "sampleCount": 389, "peakRssMb": 20113, "peakCpuPercent": 122.3, "firstAtMs": 0, "lastAtMs": 1946391 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 389, "peakUsedMb": 78221, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 121618, "totalOutputTokens": 37679, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 217700, "todayTokensCached": 197567, "todayTokensOut": 5334, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 217700, "totalTokensCached": 197567, "totalTokensOut": 5334, "totalCost": 0, "medianOutputTokensPerSec": 22.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.8, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13006, "cachedInputTokens": 12921, "outputTokens": 64, "cost": 0, "durationMs": 295990, "outputTokensPerSec": 29.38210938856511, "at": "2026-10-31T08:25:15.731Z" }, "lastUpdated": "2026-10-31T08:25:15.731Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T08:25:20.410Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1946916, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 37679, "promptTokensPerSec": null, "peakRssMb": 20113, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 217700, "output": 5334 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 479.5, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-author-linear", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 2 }, "trialId": "craftbook-author-linear-qwen3-8-27b-q4-2026-10-31T08-25-50-842Z-vtga", "success": true, "includedInCapability": true, "durationMs": 2524768, "timing": { "startedAt": "2026-10-31T08:25:50.843Z", "finishedAt": "2026-10-31T09:07:55.611Z", "timeToFirstArtifactMs": 994728, "timeToLastArtifactWriteMs": 2518787, "timeToFirstTokenMs": 50579, "firstTurnTtftMs": 48727, "timeToFirstToolCallMs": 55779 }, "metrics": { "process": { "sampleCount": 503, "peakRssMb": 50420, "peakCpuPercent": 145.7, "firstAtMs": 0, "lastAtMs": 2517956 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 503, "peakUsedMb": 98178, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 344542, "totalOutputTokens": 34574, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 211285, "todayTokensCached": 125808, "todayTokensOut": 3169, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 211285, "totalTokensCached": 125808, "totalTokensOut": 3169, "totalCost": 0, "medianOutputTokensPerSec": 26.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.2, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13365, "cachedInputTokens": 13161, "outputTokens": 56, "cost": 0, "durationMs": 566591, "outputTokensPerSec": 57.96918147515757, "at": "2026-10-31T09:07:49.630Z" }, "lastUpdated": "2026-10-31T09:07:49.630Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T09:07:51.631Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2519733, "meanTokensPerSec": 24.27, "genTokensPerSec": 24.27, "genTokens": 34574, "promptTokensPerSec": 335.19, "peakRssMb": 50420, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 211285, "output": 3169 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-route-multi", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 2 }, "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-31T09-08-17-477Z-8gh7", "success": false, "includedInCapability": true, "durationMs": 1800903, "timing": { "startedAt": "2026-10-31T09:08:17.478Z", "finishedAt": "2026-10-31T09:38:18.381Z", "timeToFirstArtifactMs": 84201, "timeToLastArtifactWriteMs": 1497083, "timeToFirstTokenMs": 15900, "firstTurnTtftMs": 13978, "timeToFirstToolCallMs": 22320 }, "metrics": { "process": { "sampleCount": 359, "peakRssMb": 30159, "peakCpuPercent": 161.7, "firstAtMs": 0, "lastAtMs": 1798908 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 359, "peakUsedMb": 80339, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 220699, "totalOutputTokens": 29295, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 21, "todayTokensIn": 279273, "todayTokensCached": 212806, "todayTokensOut": 8610, "todayCost": 0, "totalTurns": 21, "totalTokensIn": 279273, "totalTokensCached": 212806, "totalTokensOut": 8610, "totalCost": 0, "medianOutputTokensPerSec": 69.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 69.6, "turns": 21 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15139, "outputTokens": 246, "cost": 0, "durationMs": 23128, "outputTokensPerSec": 62.98203752289847, "at": "2026-10-31T09:33:14.561Z" }, "lastUpdated": "2026-10-31T09:33:14.561Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T09:38:18.231Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1799704, "meanTokensPerSec": 66.19, "genTokensPerSec": 66.19, "genTokens": 29295, "promptTokensPerSec": 988.6, "peakRssMb": 30159, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 279273, "output": 8610 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-route-multi", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 2 }, "trialId": "craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-31T09-38-41-634Z-awya", "success": false, "includedInCapability": true, "durationMs": 1679222, "timing": { "startedAt": "2026-10-31T09:38:41.634Z", "finishedAt": "2026-10-31T10:06:40.856Z", "timeToFirstArtifactMs": 219121, "timeToLastArtifactWriteMs": 1670669, "timeToFirstTokenMs": 22591, "firstTurnTtftMs": 20891, "timeToFirstToolCallMs": 28720 }, "metrics": { "process": { "sampleCount": 333, "peakRssMb": 19923, "peakCpuPercent": 121.6, "firstAtMs": 0, "lastAtMs": 1665742 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 333, "peakUsedMb": 77467, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 158650, "totalOutputTokens": 26456, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 301698, "todayTokensCached": 281447, "todayTokensOut": 10001, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 301698, "totalTokensCached": 281447, "totalTokensOut": 10001, "totalCost": 0, "medianOutputTokensPerSec": 22.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.4, "turns": 16 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9785, "cachedInputTokens": 9744, "outputTokens": 71, "cost": 0, "durationMs": 83563, "outputTokensPerSec": 25.488300842972375, "at": "2026-10-31T10:06:32.303Z" }, "lastUpdated": "2026-10-31T10:06:32.303Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T10:06:32.993Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1670292, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 26456, "promptTokensPerSec": null, "peakRssMb": 19923, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 301698, "output": 10001 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 374, "sampleCount": 16, "structuredSampleCount": 0, "firstTokenSampleCount": 16, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-route-multi", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 2 }, "trialId": "craftbook-route-multi-qwen3-8-27b-q4-2026-10-31T10-07-03-209Z-o9yo", "success": true, "includedInCapability": true, "durationMs": 1472034, "timing": { "startedAt": "2026-10-31T10:07:03.209Z", "finishedAt": "2026-10-31T10:31:35.243Z", "timeToFirstArtifactMs": 296261, "timeToLastArtifactWriteMs": 1261570, "timeToFirstTokenMs": 35635, "firstTurnTtftMs": 33723, "timeToFirstToolCallMs": 47420 }, "metrics": { "process": { "sampleCount": 293, "peakRssMb": 50953, "peakCpuPercent": 120, "firstAtMs": 1, "lastAtMs": 1464666 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 293, "peakUsedMb": 99621, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 240258, "totalOutputTokens": 17994, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 154962, "todayTokensCached": 151867, "todayTokensOut": 2680, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 154962, "totalTokensCached": 151867, "totalTokensOut": 2680, "totalCost": 0, "medianOutputTokensPerSec": 31.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.2, "turns": 10 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7382, "cachedInputTokens": 7068, "outputTokens": 66, "cost": 0, "durationMs": 198653, "outputTokensPerSec": 41.12482079068478, "at": "2026-10-31T10:31:28.742Z" }, "lastUpdated": "2026-10-31T10:31:28.742Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T10:31:32.474Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1468213, "meanTokensPerSec": 24.79, "genTokensPerSec": 24.79, "genTokens": 17994, "promptTokensPerSec": 353.59, "peakRssMb": 50953, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 154962, "output": 2680 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-route-multi", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 2 }, "trialId": "craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-31T10-31-57-276Z-cukg", "success": true, "includedInCapability": true, "durationMs": 1851100, "timing": { "startedAt": "2026-10-31T10:31:57.277Z", "finishedAt": "2026-10-31T11:02:48.377Z", "timeToFirstArtifactMs": 904312, "timeToLastArtifactWriteMs": 1816612, "timeToFirstTokenMs": 17360, "firstTurnTtftMs": 15638, "timeToFirstToolCallMs": 41708 }, "metrics": { "process": { "sampleCount": 368, "peakRssMb": 13133, "peakCpuPercent": 118.8, "firstAtMs": 0, "lastAtMs": 1841869 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 368, "peakUsedMb": 61748, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 201040, "totalOutputTokens": 53790, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 234675, "todayTokensCached": 179929, "todayTokensOut": 40930, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 234675, "totalTokensCached": 179929, "totalTokensOut": 40930, "totalCost": 0, "medianOutputTokensPerSec": 36.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.3, "turns": 22 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9063, "cachedInputTokens": 8879, "outputTokens": 33, "cost": 0, "durationMs": 21716, "outputTokensPerSec": 35.266682236505396, "at": "2026-10-31T11:02:36.074Z" }, "lastUpdated": "2026-10-31T11:02:36.074Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T11:02:41.263Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1842923, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 53790, "promptTokensPerSec": null, "peakRssMb": 13133, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 234675, "output": 40930 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1015, "sampleCount": 23, "structuredSampleCount": 0, "firstTokenSampleCount": 23, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-export-generalize", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 2 }, "trialId": "craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-31T11-03-10-332Z-nahg", "success": false, "includedInCapability": true, "durationMs": 2723865, "timing": { "startedAt": "2026-10-31T11:03:10.333Z", "finishedAt": "2026-10-31T11:48:34.198Z", "timeToFirstArtifactMs": 1226305, "timeToLastArtifactWriteMs": 2709738, "timeToFirstTokenMs": 40888, "firstTurnTtftMs": 39220, "timeToFirstToolCallMs": 191269 }, "metrics": { "process": { "sampleCount": 540, "peakRssMb": 19022, "peakCpuPercent": 155.7, "firstAtMs": 0, "lastAtMs": 2705631 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 540, "peakUsedMb": 78945, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 97927, "totalOutputTokens": 45989, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 132409, "todayTokensCached": 111642, "todayTokensOut": 12406, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 132409, "totalTokensCached": 111642, "totalTokensOut": 12406, "totalCost": 0, "medianOutputTokensPerSec": 22.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.8, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8946, "cachedInputTokens": 2036, "outputTokens": 177, "cost": 0, "durationMs": 36564, "outputTokensPerSec": 32.34587369190101, "at": "2026-10-31T11:48:20.074Z" }, "lastUpdated": "2026-10-31T11:48:20.074Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T11:48:22.055Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2710655, "meanTokensPerSec": 20, "genTokensPerSec": 20, "genTokens": 45989, "promptTokensPerSec": null, "peakRssMb": 19022, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 132409, "output": 12406 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 503, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-export-generalize", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 2 }, "trialId": "craftbook-export-generalize-qwen3-8-27b-q4-2026-10-31T11-48-55-877Z-enbw", "success": true, "includedInCapability": true, "durationMs": 4080932, "timing": { "startedAt": "2026-10-31T11:48:55.877Z", "finishedAt": "2026-10-31T12:56:56.809Z", "timeToFirstArtifactMs": 1425394, "timeToLastArtifactWriteMs": 4074126, "timeToFirstTokenMs": 50630, "firstTurnTtftMs": 48792, "timeToFirstToolCallMs": 60414 }, "metrics": { "process": { "sampleCount": 813, "peakRssMb": 50092, "peakCpuPercent": 172.1, "firstAtMs": 0, "lastAtMs": 4072716 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 813, "peakUsedMb": 98096, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 417648, "totalOutputTokens": 63023, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 299078, "todayTokensCached": 222801, "todayTokensOut": 8196, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 299078, "totalTokensCached": 222801, "totalTokensOut": 8196, "totalCost": 0, "medianOutputTokensPerSec": 33.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.8, "turns": 17 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9910, "cachedInputTokens": 9613, "outputTokens": 73, "cost": 0, "durationMs": 276175, "outputTokensPerSec": 22.445800406331337, "at": "2026-10-31T12:56:50.003Z" }, "lastUpdated": "2026-10-31T12:56:50.003Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T12:56:50.927Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4073997, "meanTokensPerSec": 22.69, "genTokensPerSec": 22.69, "genTokens": 63023, "promptTokensPerSec": 343.99, "peakRssMb": 50092, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 299078, "output": 8196 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-export-generalize", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 2 }, "trialId": "craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-31T12-57-22-612Z-hijn", "success": false, "includedInCapability": true, "durationMs": 599154, "timing": { "startedAt": "2026-10-31T12:57:22.612Z", "finishedAt": "2026-10-31T13:07:21.766Z", "timeToFirstArtifactMs": 83372, "timeToLastArtifactWriteMs": 596096, "timeToFirstTokenMs": 23209, "firstTurnTtftMs": 21544, "timeToFirstToolCallMs": 23804 }, "metrics": { "process": { "sampleCount": 119, "peakRssMb": 11039, "peakCpuPercent": 103.4, "firstAtMs": 0, "lastAtMs": 592657 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 119, "peakUsedMb": 60204, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 74723, "totalOutputTokens": 14142, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 194191, "todayTokensCached": 167930, "todayTokensOut": 6585, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 194191, "totalTokensCached": 167930, "totalTokensOut": 6585, "totalCost": 0, "medianOutputTokensPerSec": 34.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.5, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9452, "cachedInputTokens": 1269, "outputTokens": 44, "cost": 0, "durationMs": 19347, "outputTokensPerSec": 43.846891756843995, "at": "2026-10-31T13:07:18.711Z" }, "lastUpdated": "2026-10-31T13:07:18.711Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T13:07:19.864Z" } }, "source": "engine-log" }, "derived": { "durationMs": 596192, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 14142, "promptTokensPerSec": null, "peakRssMb": 11039, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 194191, "output": 6585 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 851, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-export-generalize", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 2 }, "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-31T13-07-44-345Z-27vk", "success": true, "includedInCapability": true, "durationMs": 441328, "timing": { "startedAt": "2026-10-31T13:07:44.345Z", "finishedAt": "2026-10-31T13:15:05.673Z", "timeToFirstArtifactMs": 83708, "timeToLastArtifactWriteMs": 439006, "timeToFirstTokenMs": 20625, "firstTurnTtftMs": 18756, "timeToFirstToolCallMs": 32439 }, "metrics": { "process": { "sampleCount": 88, "peakRssMb": 24989, "peakCpuPercent": 116.9, "firstAtMs": 0, "lastAtMs": 436377 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 88, "peakUsedMb": 74540, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 102998, "totalOutputTokens": 13653, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 202618, "todayTokensCached": 195382, "todayTokensOut": 1823, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 202618, "totalTokensCached": 195382, "totalTokensOut": 1823, "totalCost": 0, "medianOutputTokensPerSec": 45, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 45, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8471, "cachedInputTokens": 8037, "outputTokens": 24, "cost": 0, "durationMs": 32305, "outputTokensPerSec": 61.017668594471274, "at": "2026-10-31T13:15:03.351Z" }, "lastUpdated": "2026-10-31T13:15:03.351Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T13:15:03.791Z" } }, "source": "engine-log" }, "derived": { "durationMs": 438409, "meanTokensPerSec": 51.75, "genTokensPerSec": 51.75, "genTokens": 13653, "promptTokensPerSec": 812.35, "peakRssMb": 24989, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 202618, "output": 1823 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-author-params", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 2 }, "trialId": "craftbook-author-params-qwen3-8-27b-q4-2026-10-31T13-15-29-406Z-eutx", "success": true, "includedInCapability": true, "durationMs": 1371482, "timing": { "startedAt": "2026-10-31T13:15:29.406Z", "finishedAt": "2026-10-31T13:38:20.888Z", "timeToFirstArtifactMs": 1075579, "timeToLastArtifactWriteMs": 1365881, "timeToFirstTokenMs": 55141, "firstTurnTtftMs": 53288, "timeToFirstToolCallMs": 66061 }, "metrics": { "process": { "sampleCount": 273, "peakRssMb": 49852, "peakCpuPercent": 118.9, "firstAtMs": 0, "lastAtMs": 1364257 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 273, "peakUsedMb": 97490, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 188418, "totalOutputTokens": 15849, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 196427, "todayTokensCached": 183254, "todayTokensOut": 3777, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 196427, "totalTokensCached": 183254, "totalTokensOut": 3777, "totalCost": 0, "medianOutputTokensPerSec": 23.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.2, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 3626, "cachedInputTokens": 3551, "outputTokens": 87, "cost": 0, "durationMs": 69373, "outputTokensPerSec": 48.739931095339266, "at": "2026-10-31T13:38:08.933Z" }, "lastUpdated": "2026-10-31T13:38:08.933Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T13:38:18.758Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1368304, "meanTokensPerSec": 21.39, "genTokensPerSec": 21.39, "genTokens": 15849, "promptTokensPerSec": 324.54, "peakRssMb": 49852, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 196427, "output": 3777 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-author-params", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 2 }, "trialId": "craftbook-author-params-mlx-gemma4-12b-q4-2026-10-31T13-38-43-140Z-yyk8", "success": false, "includedInCapability": true, "durationMs": 835078, "timing": { "startedAt": "2026-10-31T13:38:43.141Z", "finishedAt": "2026-10-31T13:52:38.219Z", "timeToFirstArtifactMs": 686371, "timeToLastArtifactWriteMs": 686371, "timeToFirstTokenMs": 21022, "firstTurnTtftMs": 19335, "timeToFirstToolCallMs": 44211 }, "metrics": { "process": { "sampleCount": 166, "peakRssMb": 11556, "peakCpuPercent": 100.4, "firstAtMs": 0, "lastAtMs": 829038 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 166, "peakUsedMb": 62019, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 64163, "totalOutputTokens": 10425, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 196022, "todayTokensCached": 193887, "todayTokensOut": 1882, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 196022, "totalTokensCached": 193887, "totalTokensOut": 1882, "totalCost": 0, "medianOutputTokensPerSec": 44.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 44.2, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10192, "cachedInputTokens": 10081, "outputTokens": 131, "cost": 0, "durationMs": 113118, "outputTokensPerSec": 44.220791810521284, "at": "2026-10-31T13:51:33.944Z" }, "lastUpdated": "2026-10-31T13:51:33.944Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T13:52:36.792Z" } }, "source": "engine-log" }, "derived": { "durationMs": 832589, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 10425, "promptTokensPerSec": null, "peakRssMb": 11556, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 196022, "output": 1882 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 698, "sampleCount": 13, "structuredSampleCount": 0, "firstTokenSampleCount": 13, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-author-params", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 2 }, "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-31T13-53-01-548Z-23ms", "success": false, "includedInCapability": true, "durationMs": 4603031, "timing": { "startedAt": "2026-10-31T13:53:01.549Z", "finishedAt": "2026-10-31T15:09:44.580Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 19557, "firstTurnTtftMs": 17649, "timeToFirstToolCallMs": 27585 }, "metrics": { "process": { "sampleCount": 915, "peakRssMb": 28421, "peakCpuPercent": 102.5, "firstAtMs": 0, "lastAtMs": 4595039 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 915, "peakUsedMb": 79509, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 450149, "totalOutputTokens": 106857, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 63, "todayTokensIn": 830403, "todayTokensCached": 668313, "todayTokensOut": 51277, "todayCost": 0, "totalTurns": 63, "totalTokensIn": 830403, "totalTokensCached": 668313, "totalTokensOut": 51277, "totalCost": 0, "medianOutputTokensPerSec": 50.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 50.3, "turns": 63 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15424, "cachedInputTokens": 15071, "outputTokens": 1241, "cost": 0, "durationMs": 18940, "outputTokensPerSec": 70.40651700890858, "at": "2026-10-31T15:08:21.696Z" }, "lastUpdated": "2026-10-31T15:08:21.696Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T15:09:41.452Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4598856, "meanTokensPerSec": 51.78, "genTokensPerSec": 51.78, "genTokens": 106857, "promptTokensPerSec": 1007.75, "peakRssMb": 28421, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 830403, "output": 51277 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-author-params", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 2 }, "trialId": "craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-31T15-10-08-307Z-rq1v", "success": false, "includedInCapability": true, "durationMs": 1809679, "timing": { "startedAt": "2026-10-31T15:10:08.308Z", "finishedAt": "2026-10-31T15:40:17.987Z", "timeToFirstArtifactMs": 1126702, "timeToLastArtifactWriteMs": 1800807, "timeToFirstTokenMs": 31398, "firstTurnTtftMs": 29721, "timeToFirstToolCallMs": 41399 }, "metrics": { "process": { "sampleCount": 360, "peakRssMb": 19316, "peakCpuPercent": 115.6, "firstAtMs": 0, "lastAtMs": 1801231 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 360, "peakUsedMb": 78624, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 78991, "totalOutputTokens": 33522, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 245257, "todayTokensCached": 225327, "todayTokensOut": 13868, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 245257, "totalTokensCached": 225327, "totalTokensOut": 13868, "totalCost": 0, "medianOutputTokensPerSec": 19.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 19.4, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 46763, "cachedInputTokens": 39665, "outputTokens": 3035, "cost": 0, "durationMs": 634587, "outputTokensPerSec": 20.162239552028566, "at": "2026-10-31T15:39:29.605Z" }, "lastUpdated": "2026-10-31T15:39:29.605Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T15:40:11.132Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1801754, "meanTokensPerSec": 22, "genTokensPerSec": 22, "genTokens": 33522, "promptTokensPerSec": null, "peakRssMb": 19316, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 245257, "output": 13868 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 301, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-author-gate-script", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 2 }, "trialId": "craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-31T15-40-39-897Z-jm3h", "success": false, "includedInCapability": true, "durationMs": 576521, "timing": { "startedAt": "2026-10-31T15:40:39.898Z", "finishedAt": "2026-10-31T15:50:16.419Z", "timeToFirstArtifactMs": 250510, "timeToLastArtifactWriteMs": 571633, "timeToFirstTokenMs": 21031, "firstTurnTtftMs": 19367, "timeToFirstToolCallMs": 39052 }, "metrics": { "process": { "sampleCount": 115, "peakRssMb": 10502, "peakCpuPercent": 104.4, "firstAtMs": 0, "lastAtMs": 572311 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 115, "peakUsedMb": 62300, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 24930, "totalOutputTokens": 14753, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 152089, "todayTokensCached": 151139, "todayTokensOut": 1691, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 152089, "totalTokensCached": 151139, "totalTokensOut": 1691, "totalCost": 0, "medianOutputTokensPerSec": 36.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 24530, "cachedInputTokens": 24389, "outputTokens": 97, "cost": 0, "durationMs": 62054, "outputTokensPerSec": 43.089924632088, "at": "2026-10-31T15:50:11.531Z" }, "lastUpdated": "2026-10-31T15:50:11.531Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T15:50:15.328Z" } }, "source": "engine-log" }, "derived": { "durationMs": 574365, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 14753, "promptTokensPerSec": null, "peakRssMb": 10502, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 152089, "output": 1691 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 770, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-author-gate-script", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 2 }, "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-31T15-50-39-036Z-gsuu", "success": false, "includedInCapability": true, "durationMs": 731548, "timing": { "startedAt": "2026-10-31T15:50:39.036Z", "finishedAt": "2026-10-31T16:02:50.584Z", "timeToFirstArtifactMs": 439560, "timeToLastArtifactWriteMs": 720915, "timeToFirstTokenMs": 17152, "firstTurnTtftMs": 15253, "timeToFirstToolCallMs": 24797 }, "metrics": { "process": { "sampleCount": 145, "peakRssMb": 26419, "peakCpuPercent": 185.2, "firstAtMs": 0, "lastAtMs": 722351 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 145, "peakUsedMb": 75522, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 162221, "totalOutputTokens": 24123, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 395612, "todayTokensCached": 371317, "todayTokensOut": 6783, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 395612, "totalTokensCached": 371317, "totalTokensOut": 6783, "totalCost": 0, "medianOutputTokensPerSec": 42.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.6, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7818, "cachedInputTokens": 6895, "outputTokens": 2301, "cost": 0, "durationMs": 35667, "outputTokensPerSec": 67.22811889146678, "at": "2026-10-31T16:02:39.951Z" }, "lastUpdated": "2026-10-31T16:02:39.951Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T16:02:44.997Z" } }, "source": "engine-log" }, "derived": { "durationMs": 724909, "meanTokensPerSec": 51.16, "genTokensPerSec": 51.16, "genTokens": 24123, "promptTokensPerSec": 819.93, "peakRssMb": 26419, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 395612, "output": 6783 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-author-gate-script", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 2 }, "trialId": "craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-31T16-03-12-666Z-66bx", "success": false, "includedInCapability": true, "durationMs": 2280263, "timing": { "startedAt": "2026-10-31T16:03:12.667Z", "finishedAt": "2026-10-31T16:41:12.930Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 42161, "firstTurnTtftMs": 40494, "timeToFirstToolCallMs": 56270 }, "metrics": { "process": { "sampleCount": 453, "peakRssMb": 19093, "peakCpuPercent": 120.7, "firstAtMs": 0, "lastAtMs": 2267409 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 453, "peakUsedMb": 80678, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 131015, "totalOutputTokens": 23138, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T16:41:05.446Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2271712, "meanTokensPerSec": 12, "genTokensPerSec": 12, "genTokens": 23138, "promptTokensPerSec": null, "peakRssMb": 19093, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 533, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-author-gate-script", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 2 }, "trialId": "craftbook-author-gate-script-qwen3-8-27b-q4-2026-10-31T16-41-38-324Z-5a0c", "success": true, "includedInCapability": true, "durationMs": 5687157, "timing": { "startedAt": "2026-10-31T16:41:38.325Z", "finishedAt": "2026-10-31T18:16:25.482Z", "timeToFirstArtifactMs": 1966072, "timeToLastArtifactWriteMs": 3425433, "timeToFirstTokenMs": 51249, "firstTurnTtftMs": 49409, "timeToFirstToolCallMs": 62838 }, "metrics": { "process": { "sampleCount": 1134, "peakRssMb": 49441, "peakCpuPercent": 121.8, "firstAtMs": 0, "lastAtMs": 5683240 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1134, "peakUsedMb": 100494, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 989511, "totalOutputTokens": 34899, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 413956, "todayTokensCached": 203151, "todayTokensOut": 3234, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 413956, "totalTokensCached": 203151, "totalTokensOut": 3234, "totalCost": 0, "medianOutputTokensPerSec": 25.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 25.8, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7717, "cachedInputTokens": 7418, "outputTokens": 57, "cost": 0, "durationMs": 625690, "outputTokensPerSec": 43.48828883678923, "at": "2026-10-31T18:16:22.469Z" }, "lastUpdated": "2026-10-31T18:16:22.469Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T18:16:24.388Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5685019, "meanTokensPerSec": 20.58, "genTokensPerSec": 20.58, "genTokens": 34899, "promptTokensPerSec": 253.72, "peakRssMb": 49441, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 413956, "output": 3234 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-llamacpp-complex-work-craftbook-author-fanout", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 2 }, "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-31T18-16-47-562Z-flqy", "success": false, "includedInCapability": true, "durationMs": 1463648, "timing": { "startedAt": "2026-10-31T18:16:47.563Z", "finishedAt": "2026-10-31T18:41:11.211Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 20928, "firstTurnTtftMs": 19051, "timeToFirstToolCallMs": 32575 }, "metrics": { "process": { "sampleCount": 292, "peakRssMb": 24344, "peakCpuPercent": 115.8, "firstAtMs": 0, "lastAtMs": 1461924 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 292, "peakUsedMb": 73662, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 55748, "totalOutputTokens": 27946, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 311675, "todayTokensCached": 308387, "todayTokensOut": 5361, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 311675, "totalTokensCached": 308387, "totalTokensOut": 5361, "totalCost": 0, "medianOutputTokensPerSec": 40.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.1, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7126, "cachedInputTokens": 6890, "outputTokens": 471, "cost": 0, "durationMs": 5521, "outputTokensPerSec": 90.72763563781527, "at": "2026-10-31T18:36:05.536Z" }, "lastUpdated": "2026-10-31T18:36:05.536Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T18:41:11.072Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1462470, "meanTokensPerSec": 44.39, "genTokensPerSec": 44.39, "genTokens": 27946, "promptTokensPerSec": 760.83, "peakRssMb": 24344, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 311675, "output": 5361 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r2-qwen-mlx-complex-work-craftbook-author-fanout", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 2 }, "trialId": "craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-31T18-41-32-968Z-wq8e", "success": true, "includedInCapability": true, "durationMs": 10733240, "timing": { "startedAt": "2026-10-31T18:41:32.969Z", "finishedAt": "2026-10-31T21:40:26.209Z", "timeToFirstArtifactMs": 3090797, "timeToLastArtifactWriteMs": 10718766, "timeToFirstTokenMs": 31462, "firstTurnTtftMs": 29896, "timeToFirstToolCallMs": 36730 }, "metrics": { "process": { "sampleCount": 2136, "peakRssMb": 19906, "peakCpuPercent": 158.1, "firstAtMs": 0, "lastAtMs": 10716061 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 2136, "peakUsedMb": 90317, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 418997, "totalOutputTokens": 131873, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 742891, "todayTokensCached": 730465, "todayTokensOut": 14219, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 742891, "totalTokensCached": 730465, "totalTokensOut": 14219, "totalCost": 0, "medianOutputTokensPerSec": 17.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 17.8, "turns": 14 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7602, "cachedInputTokens": 7520, "outputTokens": 47, "cost": 0, "durationMs": 248497, "outputTokensPerSec": 30.43367994436324, "at": "2026-10-31T21:40:11.734Z" }, "lastUpdated": "2026-10-31T21:40:11.734Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T21:40:14.047Z" } }, "source": "engine-log" }, "derived": { "durationMs": 10720116, "meanTokensPerSec": 13, "genTokensPerSec": 13, "genTokens": 131873, "promptTokensPerSec": null, "peakRssMb": 19906, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 742891, "output": 14219 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 399, "sampleCount": 16, "structuredSampleCount": 0, "firstTokenSampleCount": 16, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r2-qwen-llamacpp-complex-work-craftbook-author-fanout", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 2 }, "trialId": "craftbook-author-fanout-qwen3-8-27b-q4-2026-10-31T21-40-48-720Z-d3ei", "success": false, "includedInCapability": true, "durationMs": 3059910, "timing": { "startedAt": "2026-10-31T21:40:48.720Z", "finishedAt": "2026-10-31T22:31:48.630Z", "timeToFirstArtifactMs": 1290701, "timeToLastArtifactWriteMs": 1971565, "timeToFirstTokenMs": 52180, "firstTurnTtftMs": 50353, "timeToFirstToolCallMs": 63884 }, "metrics": { "process": { "sampleCount": 609, "peakRssMb": 49449, "peakCpuPercent": 104.5, "firstAtMs": 0, "lastAtMs": 3049975 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 609, "peakUsedMb": 102017, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 285509, "totalOutputTokens": 29271, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 67605, "todayTokensCached": 67075, "todayTokensOut": 1685, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 67605, "totalTokensCached": 67075, "totalTokensOut": 1685, "totalCost": 0, "medianOutputTokensPerSec": 22.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 22.4, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 59755, "cachedInputTokens": 59582, "outputTokens": 1104, "cost": 0, "durationMs": 1368076, "outputTokensPerSec": 14.257326307440126, "at": "2026-10-31T22:03:38.623Z" }, "lastUpdated": "2026-10-31T22:03:38.623Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T22:31:42.288Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3052514, "meanTokensPerSec": 18.61, "genTokensPerSec": 18.61, "genTokens": 29271, "promptTokensPerSec": 271.89, "peakRssMb": 49449, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 67605, "output": 1685 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r2-gemma-mlx-complex-work-craftbook-author-fanout", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 2 }, "trialId": "craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-31T22-32-10-078Z-dekr", "success": false, "includedInCapability": true, "durationMs": 1258841, "timing": { "startedAt": "2026-10-31T22:32:10.079Z", "finishedAt": "2026-10-31T22:53:08.920Z", "timeToFirstArtifactMs": 320451, "timeToLastArtifactWriteMs": 1067793, "timeToFirstTokenMs": 20900, "firstTurnTtftMs": 19232, "timeToFirstToolCallMs": 45295 }, "metrics": { "process": { "sampleCount": 251, "peakRssMb": 13174, "peakCpuPercent": 141.4, "firstAtMs": 0, "lastAtMs": 1255779 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 251, "peakUsedMb": 63851, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 59737, "totalOutputTokens": 24299, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 360675, "todayTokensCached": 355425, "todayTokensOut": 4963, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 360675, "totalTokensCached": 355425, "totalTokensOut": 4963, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 16 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10075, "cachedInputTokens": 10004, "outputTokens": 125, "cost": 0, "durationMs": 37808, "outputTokensPerSec": 45.897110613518116, "at": "2026-10-31T22:50:07.256Z" }, "lastUpdated": "2026-10-31T22:50:07.256Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T22:53:07.464Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1256319, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 24299, "promptTokensPerSec": null, "peakRssMb": 13174, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 360675, "output": 4963 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 481, "sampleCount": 18, "structuredSampleCount": 0, "firstTokenSampleCount": 18, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-core-tictactoe", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "tictactoe", "repetition": 3 }, "trialId": "tictactoe-mlx-gemma4-12b-q4-2026-10-31T22-53-30-091Z-127v", "success": false, "includedInCapability": true, "durationMs": 514871, "timing": { "startedAt": "2026-10-31T22:53:30.091Z", "finishedAt": "2026-10-31T23:02:04.962Z", "timeToFirstArtifactMs": 38721, "timeToLastArtifactWriteMs": 315810, "timeToFirstTokenMs": 13388, "firstTurnTtftMs": 11714, "timeToFirstToolCallMs": 25354 }, "metrics": { "process": { "sampleCount": 102, "peakRssMb": 11040, "peakCpuPercent": 196.8, "firstAtMs": 1, "lastAtMs": 507095 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 102, "peakUsedMb": 64314, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 67043, "totalOutputTokens": 12422, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 41642, "todayTokensCached": 33023, "todayTokensOut": 239, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 41642, "totalTokensCached": 33023, "totalTokensOut": 239, "totalCost": 0, "medianOutputTokensPerSec": 46.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 46.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13233, "cachedInputTokens": 13005, "outputTokens": 31, "cost": 0, "durationMs": 159185, "outputTokensPerSec": 32.80197774565829, "at": "2026-10-31T22:58:45.900Z" }, "lastUpdated": "2026-10-31T22:58:45.900Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:02:01.044Z" } }, "source": "engine-log" }, "derived": { "durationMs": 509890, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 12422, "promptTokensPerSec": null, "peakRssMb": 11040, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 41642, "output": 239 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1338, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-tictactoe", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "tictactoe", "repetition": 3 }, "trialId": "tictactoe-gemma4-12b-q4-2026-10-31T23-02-51-737Z-gi6h", "success": true, "includedInCapability": true, "durationMs": 45956, "timing": { "startedAt": "2026-10-31T23:02:51.737Z", "finishedAt": "2026-10-31T23:03:37.693Z", "timeToFirstArtifactMs": 37980, "timeToLastArtifactWriteMs": 37984, "timeToFirstTokenMs": 11036, "firstTurnTtftMs": 9233, "timeToFirstToolCallMs": 15474 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 18063, "peakCpuPercent": 119.9, "firstAtMs": 0, "lastAtMs": 40129 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 66642, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13565, "totalOutputTokens": 1864, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6952, "todayTokensCached": 6777, "todayTokensOut": 24, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6952, "totalTokensCached": 6777, "totalTokensOut": 24, "totalCost": 0, "medianOutputTokensPerSec": 97.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 97.7, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6952, "cachedInputTokens": 6777, "outputTokens": 24, "cost": 0, "durationMs": 14865, "outputTokensPerSec": 97.6686908148966, "at": "2026-10-31T23:03:08.405Z" }, "lastUpdated": "2026-10-31T23:03:08.405Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:03:35.092Z" } }, "source": "engine-log" }, "derived": { "durationMs": 42412, "meanTokensPerSec": 125.57, "genTokensPerSec": 125.57, "genTokens": 1864, "promptTokensPerSec": 1299.76, "peakRssMb": 18063, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6952, "output": 24 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-tictactoe", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "tictactoe", "repetition": 3 }, "trialId": "tictactoe-mlx-qwen3-8-27b-q4-2026-10-31T23-04-26-418Z-z6yq", "success": true, "includedInCapability": true, "durationMs": 99266, "timing": { "startedAt": "2026-10-31T23:04:26.419Z", "finishedAt": "2026-10-31T23:06:05.685Z", "timeToFirstArtifactMs": 88875, "timeToLastArtifactWriteMs": 88885, "timeToFirstTokenMs": 22137, "firstTurnTtftMs": 20458, "timeToFirstToolCallMs": 28262 }, "metrics": { "process": { "sampleCount": 19, "peakRssMb": 18231, "peakCpuPercent": 118.1, "firstAtMs": 0, "lastAtMs": 90300 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 19, "peakUsedMb": 80085, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18655, "totalOutputTokens": 1508, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 29037, "todayTokensCached": 23378, "todayTokensOut": 1754, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 29037, "totalTokensCached": 23378, "totalTokensOut": 1754, "totalCost": 0, "medianOutputTokensPerSec": 41.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 41.7, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5321, "cachedInputTokens": 0, "outputTokens": 1527, "cost": 0, "durationMs": 37314, "outputTokensPerSec": 50.405393747730564, "at": "2026-10-31T23:05:55.304Z" }, "lastUpdated": "2026-10-31T23:05:55.304Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:05:59.817Z" } }, "source": "engine-log" }, "derived": { "durationMs": 92332, "meanTokensPerSec": 53, "genTokensPerSec": 53, "genTokens": 1508, "promptTokensPerSec": null, "peakRssMb": 18231, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 29037, "output": 1754 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 544.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-tictactoe", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "tictactoe", "repetition": 3 }, "trialId": "tictactoe-qwen3-8-27b-q4-2026-10-31T23-06-49-606Z-5b0x", "success": true, "includedInCapability": true, "durationMs": 142537, "timing": { "startedAt": "2026-10-31T23:06:49.607Z", "finishedAt": "2026-10-31T23:09:12.144Z", "timeToFirstArtifactMs": 135323, "timeToLastArtifactWriteMs": 135323, "timeToFirstTokenMs": 21091, "firstTurnTtftMs": 19343, "timeToFirstToolCallMs": 28587 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 40975, "peakCpuPercent": 120.3, "firstAtMs": 0, "lastAtMs": 135447 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 89702, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13461, "totalOutputTokens": 3665, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22962, "todayTokensCached": 21715, "todayTokensOut": 519, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22962, "totalTokensCached": 21715, "totalTokensOut": 519, "totalCost": 0, "medianOutputTokensPerSec": 37.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.5, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8148, "cachedInputTokens": 7871, "outputTokens": 115, "cost": 0, "durationMs": 29153, "outputTokensPerSec": 30.440465525505243, "at": "2026-10-31T23:08:00.721Z" }, "lastUpdated": "2026-10-31T23:08:00.721Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:09:07.782Z" } }, "source": "engine-log" }, "derived": { "durationMs": 137228, "meanTokensPerSec": 38.02, "genTokensPerSec": 38.02, "genTokens": 3665, "promptTokensPerSec": 500.31, "peakRssMb": 40975, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22962, "output": 519 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-core-petshop", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "petshop", "repetition": 3 }, "trialId": "petshop-gemma4-12b-q4-2026-10-31T23-09-58-004Z-0irx", "success": false, "includedInCapability": true, "durationMs": 604846, "timing": { "startedAt": "2026-10-31T23:09:58.005Z", "finishedAt": "2026-10-31T23:20:02.851Z", "timeToFirstArtifactMs": 113549, "timeToLastArtifactWriteMs": 532242, "timeToFirstTokenMs": 12182, "firstTurnTtftMs": 10307, "timeToFirstToolCallMs": 20997 }, "metrics": { "process": { "sampleCount": 120, "peakRssMb": 27506, "peakCpuPercent": 106.8, "firstAtMs": 0, "lastAtMs": 596963 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 120, "peakUsedMb": 78627, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 153584, "totalOutputTokens": 23708, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 62496, "todayTokensCached": 37757, "todayTokensOut": 6195, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 62496, "totalTokensCached": 37757, "totalTokensOut": 6195, "totalCost": 0, "medianOutputTokensPerSec": 66.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 66.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12583, "outputTokens": 2005, "cost": 0, "durationMs": 112767, "outputTokensPerSec": 77.49013924444408, "at": "2026-10-31T23:16:39.703Z" }, "lastUpdated": "2026-10-31T23:16:39.703Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:19:56.554Z" } }, "source": "engine-log" }, "derived": { "durationMs": 597496, "meanTokensPerSec": 67.43, "genTokensPerSec": 67.43, "genTokens": 23708, "promptTokensPerSec": 893.57, "peakRssMb": 27506, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 62496, "output": 6195 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-petshop", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "petshop", "repetition": 3 }, "trialId": "petshop-mlx-qwen3-8-27b-q4-2026-10-31T23-20-23-986Z-iqxn", "success": false, "includedInCapability": true, "durationMs": 1480921, "timing": { "startedAt": "2026-10-31T23:20:23.987Z", "finishedAt": "2026-10-31T23:45:04.908Z", "timeToFirstArtifactMs": 406535, "timeToLastArtifactWriteMs": 1463459, "timeToFirstTokenMs": 31575, "firstTurnTtftMs": 29905, "timeToFirstToolCallMs": 37708 }, "metrics": { "process": { "sampleCount": 293, "peakRssMb": 26861, "peakCpuPercent": 299.3, "firstAtMs": 0, "lastAtMs": 1465145 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 293, "peakUsedMb": 88424, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 66520, "totalOutputTokens": 30890, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 67967, "todayTokensCached": 66325, "todayTokensOut": 2709, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 67967, "totalTokensCached": 66325, "totalTokensOut": 2709, "totalCost": 0, "medianOutputTokensPerSec": 23.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.7, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13105, "cachedInputTokens": 12158, "outputTokens": 1061, "cost": 0, "durationMs": 74182, "outputTokensPerSec": 14.926501099105735, "at": "2026-10-31T23:38:42.872Z" }, "lastUpdated": "2026-10-31T23:38:42.872Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:44:50.748Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1465691, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 30890, "promptTokensPerSec": null, "peakRssMb": 26861, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 67967, "output": 2709 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 483.5, "sampleCount": 8, "structuredSampleCount": 0, "firstTokenSampleCount": 8, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-petshop", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "petshop", "repetition": 3 }, "trialId": "petshop-qwen3-8-27b-q4-2026-10-31T23-45-29-971Z-10s6", "success": true, "includedInCapability": true, "durationMs": 512258, "timing": { "startedAt": "2026-10-31T23:45:29.972Z", "finishedAt": "2026-10-31T23:54:02.230Z", "timeToFirstArtifactMs": 394248, "timeToLastArtifactWriteMs": 394248, "timeToFirstTokenMs": 25450, "firstTurnTtftMs": 23614, "timeToFirstToolCallMs": 39150 }, "metrics": { "process": { "sampleCount": 100, "peakRssMb": 54716, "peakCpuPercent": 105.9, "firstAtMs": 1, "lastAtMs": 496641 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 100, "peakUsedMb": 109145, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 51768, "totalOutputTokens": 8299, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 50203, "todayTokensCached": 48140, "todayTokensOut": 1111, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 50203, "totalTokensCached": 48140, "totalTokensOut": 1111, "totalCost": 0, "medianOutputTokensPerSec": 28.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.2, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 6957, "cachedInputTokens": 6369, "outputTokens": 123, "cost": 0, "durationMs": 309370, "outputTokensPerSec": 31.504684385038892, "at": "2026-10-31T23:52:12.330Z" }, "lastUpdated": "2026-10-31T23:52:12.330Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-31T23:53:48.689Z" } }, "source": "engine-log" }, "derived": { "durationMs": 497667, "meanTokensPerSec": 28.23, "genTokensPerSec": 28.23, "genTokens": 8299, "promptTokensPerSec": 388.55, "peakRssMb": 54716, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 50203, "output": 1111 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-petshop", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "petshop", "repetition": 3 }, "trialId": "petshop-mlx-gemma4-12b-q4-2026-10-31T23-54-38-681Z-5v0l", "success": false, "includedInCapability": true, "durationMs": 1646243, "timing": { "startedAt": "2026-10-31T23:54:38.682Z", "finishedAt": "2026-11-01T00:22:04.925Z", "timeToFirstArtifactMs": 84470, "timeToLastArtifactWriteMs": 1639052, "timeToFirstTokenMs": 17184, "firstTurnTtftMs": 15501, "timeToFirstToolCallMs": 34228 }, "metrics": { "process": { "sampleCount": 327, "peakRssMb": 12672, "peakCpuPercent": 109.4, "firstAtMs": 0, "lastAtMs": 1638282 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 327, "peakUsedMb": 64329, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 152614, "totalOutputTokens": 56284, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 26, "todayTokensIn": 319589, "todayTokensCached": 287704, "todayTokensOut": 31393, "todayCost": 0, "totalTurns": 29, "totalTokensIn": 350475, "totalTokensCached": 313293, "totalTokensOut": 31761, "totalCost": 0, "medianOutputTokensPerSec": 34, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34, "turns": 29 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11931, "cachedInputTokens": 1367, "outputTokens": 1273, "cost": 0, "durationMs": 48518, "outputTokensPerSec": 33.89990669619816, "at": "2026-11-01T00:21:57.733Z" }, "lastUpdated": "2026-11-01T00:21:57.733Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:21:58.561Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1638811, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 56284, "promptTokensPerSec": null, "peakRssMb": 12672, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 350475, "output": 31761 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 603, "sampleCount": 34, "structuredSampleCount": 0, "firstTokenSampleCount": 34, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-core-tankcombat", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "tankcombat", "repetition": 3 }, "trialId": "tankcombat-mlx-qwen3-8-27b-q4-2026-11-01T00-22-26-542Z-pzr8", "success": true, "includedInCapability": true, "durationMs": 250022, "timing": { "startedAt": "2026-11-01T00:22:26.543Z", "finishedAt": "2026-11-01T00:26:36.565Z", "timeToFirstArtifactMs": 239216, "timeToLastArtifactWriteMs": 239216, "timeToFirstTokenMs": 30541, "firstTurnTtftMs": 28853, "timeToFirstToolCallMs": 32508 }, "metrics": { "process": { "sampleCount": 49, "peakRssMb": 18661, "peakCpuPercent": 137, "firstAtMs": 0, "lastAtMs": 240806 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 81104, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16856, "totalOutputTokens": 5615, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12281, "todayTokensCached": 12109, "todayTokensOut": 636, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12281, "totalTokensCached": 12109, "totalTokensOut": 636, "totalCost": 0, "medianOutputTokensPerSec": 28.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.2, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12281, "cachedInputTokens": 12109, "outputTokens": 636, "cost": 0, "durationMs": 77806, "outputTokensPerSec": 28.239434784904503, "at": "2026-11-01T00:23:46.037Z" }, "lastUpdated": "2026-11-01T00:23:46.037Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:26:30.959Z" } }, "source": "engine-log" }, "derived": { "durationMs": 243343, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 5615, "promptTokensPerSec": null, "peakRssMb": 18661, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12281, "output": 636 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 544.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-tankcombat", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "tankcombat", "repetition": 3 }, "trialId": "tankcombat-qwen3-8-27b-q4-2026-11-01T00-27-20-468Z-u26i", "success": true, "includedInCapability": true, "durationMs": 162861, "timing": { "startedAt": "2026-11-01T00:27:20.468Z", "finishedAt": "2026-11-01T00:30:03.329Z", "timeToFirstArtifactMs": 154177, "timeToLastArtifactWriteMs": 154181, "timeToFirstTokenMs": 24451, "firstTurnTtftMs": 22591, "timeToFirstToolCallMs": 33580 }, "metrics": { "process": { "sampleCount": 32, "peakRssMb": 39323, "peakCpuPercent": 108.8, "firstAtMs": 0, "lastAtMs": 155518 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 87785, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13145, "totalOutputTokens": 4293, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15328, "todayTokensCached": 14290, "todayTokensOut": 601, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15328, "totalTokensCached": 14290, "totalTokensOut": 601, "totalCost": 0, "medianOutputTokensPerSec": 28.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.7, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7766, "cachedInputTokens": 7145, "outputTokens": 351, "cost": 0, "durationMs": 13635, "outputTokensPerSec": 30.226088551558068, "at": "2026-11-01T00:28:18.230Z" }, "lastUpdated": "2026-11-01T00:28:18.230Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:29:59.342Z" } }, "source": "engine-log" }, "derived": { "durationMs": 157822, "meanTokensPerSec": 38.59, "genTokensPerSec": 38.59, "genTokens": 4293, "promptTokensPerSec": 458.29, "peakRssMb": 39323, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15328, "output": 601 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-tankcombat", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "tankcombat", "repetition": 3 }, "trialId": "tankcombat-mlx-gemma4-12b-q4-2026-11-01T00-30-52-296Z-lgzo", "success": true, "includedInCapability": true, "durationMs": 101405, "timing": { "startedAt": "2026-11-01T00:30:52.297Z", "finishedAt": "2026-11-01T00:32:33.702Z", "timeToFirstArtifactMs": 95175, "timeToLastArtifactWriteMs": 95184, "timeToFirstTokenMs": 16389, "firstTurnTtftMs": 14604, "timeToFirstToolCallMs": 45906 }, "metrics": { "process": { "sampleCount": 20, "peakRssMb": 11215, "peakCpuPercent": 115.1, "firstAtMs": 0, "lastAtMs": 95358 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 64516, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17044, "totalOutputTokens": 2552, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 17012, "todayTokensCached": 10721, "todayTokensOut": 1707, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 17012, "totalTokensCached": 10721, "totalTokensOut": 1707, "totalCost": 0, "medianOutputTokensPerSec": 42.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6115, "cachedInputTokens": 0, "outputTokens": 1684, "cost": 0, "durationMs": 43445, "outputTokensPerSec": 42.46544378986581, "at": "2026-11-01T00:32:27.481Z" }, "lastUpdated": "2026-11-01T00:32:27.481Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:32:31.125Z" } }, "source": "engine-log" }, "derived": { "durationMs": 97665, "meanTokensPerSec": 43, "genTokensPerSec": 43, "genTokens": 2552, "promptTokensPerSec": null, "peakRssMb": 11215, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 17012, "output": 1707 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1463.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-tankcombat", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "tankcombat", "repetition": 3 }, "trialId": "tankcombat-gemma4-12b-q4-2026-11-01T00-33-23-075Z-qmie", "success": true, "includedInCapability": true, "durationMs": 73715, "timing": { "startedAt": "2026-11-01T00:33:23.075Z", "finishedAt": "2026-11-01T00:34:36.790Z", "timeToFirstArtifactMs": 65148, "timeToLastArtifactWriteMs": 65152, "timeToFirstTokenMs": 11145, "firstTurnTtftMs": 9248, "timeToFirstToolCallMs": 18492 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 18219, "peakCpuPercent": 93.5, "firstAtMs": 1, "lastAtMs": 65207 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 67688, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15630, "totalOutputTokens": 4723, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 13553, "todayTokensCached": 7225, "todayTokensOut": 2031, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 13553, "totalTokensCached": 7225, "totalTokensOut": 2031, "totalCost": 0, "medianOutputTokensPerSec": 96.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 96.9, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6156, "outputTokens": 2008, "cost": 0, "durationMs": 43706, "outputTokensPerSec": 116.10716454097607, "at": "2026-11-01T00:34:28.227Z" }, "lastUpdated": "2026-11-01T00:34:28.227Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:34:31.630Z" } }, "source": "engine-log" }, "derived": { "durationMs": 67506, "meanTokensPerSec": 110.81, "genTokensPerSec": 110.81, "genTokens": 4723, "promptTokensPerSec": 1271.71, "peakRssMb": 18219, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 13553, "output": 2031 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-core-schema-migration", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "schema-migration", "repetition": 3 }, "trialId": "schema-migration-qwen3-8-27b-q4-2026-11-01T00-35-22-226Z-hzmg", "success": true, "includedInCapability": true, "durationMs": 195314, "timing": { "startedAt": "2026-11-01T00:35:22.227Z", "finishedAt": "2026-11-01T00:38:37.541Z", "timeToFirstArtifactMs": 35099, "timeToLastArtifactWriteMs": 187292, "timeToFirstTokenMs": 16110, "firstTurnTtftMs": 14286, "timeToFirstToolCallMs": 17594 }, "metrics": { "process": { "sampleCount": 39, "peakRssMb": 44712, "peakCpuPercent": 96.5, "firstAtMs": 0, "lastAtMs": 190641 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 39, "peakUsedMb": 89781, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 46126, "totalOutputTokens": 3226, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 28115, "todayTokensCached": 9172, "todayTokensOut": 1470, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 28115, "totalTokensCached": 9172, "totalTokensOut": 1470, "totalCost": 0, "medianOutputTokensPerSec": 57, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 57, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8250, "outputTokens": 372, "cost": 0, "durationMs": 49915, "outputTokensPerSec": 49.43734043631984, "at": "2026-11-01T00:37:23.840Z" }, "lastUpdated": "2026-11-01T00:37:23.840Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:38:34.943Z" } }, "source": "engine-log" }, "derived": { "durationMs": 191665, "meanTokensPerSec": 39.55, "genTokensPerSec": 39.55, "genTokens": 3226, "promptTokensPerSec": 498, "peakRssMb": 44712, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28115, "output": 1470 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-schema-migration", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "schema-migration", "repetition": 3 }, "trialId": "schema-migration-mlx-gemma4-12b-q4-2026-11-01T00-38-59-115Z-0pur", "success": true, "includedInCapability": true, "durationMs": 344596, "timing": { "startedAt": "2026-11-01T00:38:59.115Z", "finishedAt": "2026-11-01T00:44:43.711Z", "timeToFirstArtifactMs": 36518, "timeToLastArtifactWriteMs": 339489, "timeToFirstTokenMs": 13038, "firstTurnTtftMs": 11393, "timeToFirstToolCallMs": 26145 }, "metrics": { "process": { "sampleCount": 69, "peakRssMb": 10705, "peakCpuPercent": 100.5, "firstAtMs": 0, "lastAtMs": 341423 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 69, "peakUsedMb": 65329, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 122286, "totalOutputTokens": 4781, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 122410, "todayTokensCached": 67698, "todayTokensOut": 2398, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 122410, "totalTokensCached": 67698, "totalTokensOut": 2398, "totalCost": 0, "medianOutputTokensPerSec": 34.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.7, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14945, "cachedInputTokens": 14824, "outputTokens": 208, "cost": 0, "durationMs": 36537, "outputTokensPerSec": 31.77301460593815, "at": "2026-11-01T00:44:38.610Z" }, "lastUpdated": "2026-11-01T00:44:38.610Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:44:42.883Z" } }, "source": "engine-log" }, "derived": { "durationMs": 342700, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 4781, "promptTokensPerSec": null, "peakRssMb": 10705, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 122410, "output": 2398 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1011.5, "sampleCount": 12, "structuredSampleCount": 0, "firstTokenSampleCount": 12, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-schema-migration", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "schema-migration", "repetition": 3 }, "trialId": "schema-migration-gemma4-12b-q4-2026-11-01T00-45-06-495Z-4b1k", "success": true, "includedInCapability": true, "durationMs": 174943, "timing": { "startedAt": "2026-11-01T00:45:06.496Z", "finishedAt": "2026-11-01T00:48:01.439Z", "timeToFirstArtifactMs": 27164, "timeToLastArtifactWriteMs": 167380, "timeToFirstTokenMs": 10711, "firstTurnTtftMs": 8860, "timeToFirstToolCallMs": 15008 }, "metrics": { "process": { "sampleCount": 34, "peakRssMb": 25393, "peakCpuPercent": 88.1, "firstAtMs": 0, "lastAtMs": 165621 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 69106, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 59919, "totalOutputTokens": 5517, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 82664, "todayTokensCached": 37750, "todayTokensOut": 2932, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 82664, "totalTokensCached": 37750, "totalTokensOut": 2932, "totalCost": 0, "medianOutputTokensPerSec": 84.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 84.1, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12900, "cachedInputTokens": 12462, "outputTokens": 33, "cost": 0, "durationMs": 17124, "outputTokensPerSec": 46.0701446746512, "at": "2026-11-01T00:47:53.876Z" }, "lastUpdated": "2026-11-01T00:47:53.876Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:47:54.199Z" } }, "source": "engine-log" }, "derived": { "durationMs": 166651, "meanTokensPerSec": 71.84, "genTokensPerSec": 71.84, "genTokens": 5517, "promptTokensPerSec": 959.84, "peakRssMb": 25393, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 82664, "output": 2932 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-schema-migration", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "schema-migration", "repetition": 3 }, "trialId": "schema-migration-mlx-qwen3-8-27b-q4-2026-11-01T00-48-22-682Z-llwn", "success": true, "includedInCapability": true, "durationMs": 194130, "timing": { "startedAt": "2026-11-01T00:48:22.682Z", "finishedAt": "2026-11-01T00:51:36.812Z", "timeToFirstArtifactMs": 30441, "timeToLastArtifactWriteMs": 189509, "timeToFirstTokenMs": 16193, "firstTurnTtftMs": 14542, "timeToFirstToolCallMs": 17919 }, "metrics": { "process": { "sampleCount": 39, "peakRssMb": 18161, "peakCpuPercent": 111.1, "firstAtMs": 0, "lastAtMs": 190692 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 39, "peakUsedMb": 81094, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 39287, "totalOutputTokens": 2497, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 38134, "todayTokensCached": 8153, "todayTokensOut": 2418, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 38134, "totalTokensCached": 8153, "totalTokensOut": 2418, "totalCost": 0, "medianOutputTokensPerSec": 37.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.6, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7777, "cachedInputTokens": 0, "outputTokens": 890, "cost": 0, "durationMs": 42308, "outputTokensPerSec": 31.889032576392925, "at": "2026-11-01T00:51:32.191Z" }, "lastUpdated": "2026-11-01T00:51:32.191Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:51:35.492Z" } }, "source": "engine-log" }, "derived": { "durationMs": 191740, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 2497, "promptTokensPerSec": null, "peakRssMb": 18161, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 38134, "output": 2418 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 540, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-core-failing-tests-spec", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "failing-tests-spec", "repetition": 3 }, "trialId": "failing-tests-spec-mlx-gemma4-12b-q4-2026-11-01T00-52-01-218Z-qqzj", "success": false, "includedInCapability": true, "durationMs": 270052, "timing": { "startedAt": "2026-11-01T00:52:01.219Z", "finishedAt": "2026-11-01T00:56:31.271Z", "timeToFirstArtifactMs": 70130, "timeToLastArtifactWriteMs": 264922, "timeToFirstTokenMs": 12212, "firstTurnTtftMs": 10435, "timeToFirstToolCallMs": 12660 }, "metrics": { "process": { "sampleCount": 54, "peakRssMb": 10808, "peakCpuPercent": 109.6, "firstAtMs": 0, "lastAtMs": 266022 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 65322, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 91752, "totalOutputTokens": 4538, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 51585, "todayTokensCached": 28530, "todayTokensOut": 2331, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 51585, "totalTokensCached": 28530, "totalTokensOut": 2331, "totalCost": 0, "medianOutputTokensPerSec": 35.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17035, "cachedInputTokens": 15976, "outputTokens": 521, "cost": 0, "durationMs": 16143, "outputTokensPerSec": 34.30676063424082, "at": "2026-11-01T00:56:26.141Z" }, "lastUpdated": "2026-11-01T00:56:26.141Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:56:29.953Z" } }, "source": "engine-log" }, "derived": { "durationMs": 267569, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 4538, "promptTokensPerSec": null, "peakRssMb": 10808, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 51585, "output": 2331 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1131.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-failing-tests-spec", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "failing-tests-spec", "repetition": 3 }, "trialId": "failing-tests-spec-gemma4-12b-q4-2026-11-01T00-56-53-213Z-6nwv", "success": true, "includedInCapability": true, "durationMs": 34397, "timing": { "startedAt": "2026-11-01T00:56:53.214Z", "finishedAt": "2026-11-01T00:57:27.611Z", "timeToFirstArtifactMs": 29154, "timeToLastArtifactWriteMs": 29155, "timeToFirstTokenMs": 10529, "firstTurnTtftMs": 8675, "timeToFirstToolCallMs": 11838 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 16548, "peakCpuPercent": 105.5, "firstAtMs": 0, "lastAtMs": 30103 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 61485, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13669, "totalOutputTokens": 802, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6623, "todayTokensCached": 0, "todayTokensOut": 658, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6623, "totalTokensCached": 0, "totalTokensOut": 658, "totalCost": 0, "medianOutputTokensPerSec": 83.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 83.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6623, "outputTokens": 658, "cost": 0, "durationMs": 27301, "outputTokensPerSec": 83.41950386091366, "at": "2026-11-01T00:57:22.369Z" }, "lastUpdated": "2026-11-01T00:57:22.369Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:57:25.669Z" } }, "source": "engine-log" }, "derived": { "durationMs": 31401, "meanTokensPerSec": 85.39, "genTokensPerSec": 85.39, "genTokens": 802, "promptTokensPerSec": 969.8, "peakRssMb": 16548, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6623, "output": 658 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-failing-tests-spec", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "failing-tests-spec", "repetition": 3 }, "trialId": "failing-tests-spec-mlx-qwen3-8-27b-q4-2026-11-01T00-57-49-886Z-4wf3", "success": true, "includedInCapability": true, "durationMs": 38348, "timing": { "startedAt": "2026-11-01T00:57:49.887Z", "finishedAt": "2026-11-01T00:58:28.235Z", "timeToFirstArtifactMs": 36034, "timeToLastArtifactWriteMs": 36034, "timeToFirstTokenMs": 14447, "firstTurnTtftMs": 12767, "timeToFirstToolCallMs": 16098 }, "metrics": { "process": { "sampleCount": 8, "peakRssMb": 17980, "peakCpuPercent": 122.7, "firstAtMs": 0, "lastAtMs": 35117 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 81164, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8476, "totalOutputTokens": 417, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4916, "todayTokensCached": 0, "todayTokensOut": 449, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4916, "totalTokensCached": 0, "totalTokensOut": 449, "totalCost": 0, "medianOutputTokensPerSec": 40.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 40.8, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4916, "cachedInputTokens": 0, "outputTokens": 449, "cost": 0, "durationMs": 34361, "outputTokensPerSec": 40.75722125969952, "at": "2026-11-01T00:58:25.928Z" }, "lastUpdated": "2026-11-01T00:58:25.928Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:58:27.362Z" } }, "source": "engine-log" }, "derived": { "durationMs": 36406, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 417, "promptTokensPerSec": null, "peakRssMb": 17980, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 4916, "output": 449 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 578, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-failing-tests-spec", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "failing-tests-spec", "repetition": 3 }, "trialId": "failing-tests-spec-qwen3-8-27b-q4-2026-11-01T00-58-50-594Z-ueyr", "success": true, "includedInCapability": true, "durationMs": 39021, "timing": { "startedAt": "2026-11-01T00:58:50.594Z", "finishedAt": "2026-11-01T00:59:29.615Z", "timeToFirstArtifactMs": 33681, "timeToLastArtifactWriteMs": 33683, "timeToFirstTokenMs": 15610, "firstTurnTtftMs": 13789, "timeToFirstToolCallMs": 16355 }, "metrics": { "process": { "sampleCount": 8, "peakRssMb": 37704, "peakCpuPercent": 118.2, "firstAtMs": 0, "lastAtMs": 35120 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 84677, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 8646, "totalOutputTokens": 528, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4963, "todayTokensCached": 0, "todayTokensOut": 471, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4963, "totalTokensCached": 0, "totalTokensOut": 471, "totalCost": 0, "medianOutputTokensPerSec": 55.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 55.4, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4963, "outputTokens": 471, "cost": 0, "durationMs": 31862, "outputTokensPerSec": 55.37553923404082, "at": "2026-11-01T00:59:24.277Z" }, "lastUpdated": "2026-11-01T00:59:24.277Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T00:59:28.064Z" } }, "source": "engine-log" }, "derived": { "durationMs": 36418, "meanTokensPerSec": 56.38, "genTokensPerSec": 56.38, "genTokens": 528, "promptTokensPerSec": 547.67, "peakRssMb": 37704, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4963, "output": 471 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-core-symptom-debug", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "symptom-debug", "repetition": 3 }, "trialId": "symptom-debug-gemma4-12b-q4-2026-11-01T00-59-50-716Z-y126", "success": true, "includedInCapability": true, "durationMs": 48908, "timing": { "startedAt": "2026-11-01T00:59:50.717Z", "finishedAt": "2026-11-01T01:00:39.625Z", "timeToFirstArtifactMs": 46151, "timeToLastArtifactWriteMs": 46151, "timeToFirstTokenMs": 9473, "firstTurnTtftMs": 7691, "timeToFirstToolCallMs": 13287 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 21705, "peakCpuPercent": 110.9, "firstAtMs": 0, "lastAtMs": 45145 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 66567, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 14294, "totalOutputTokens": 2655, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7256, "todayTokensCached": 0, "todayTokensOut": 2153, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7256, "totalTokensCached": 0, "totalTokensOut": 2153, "totalCost": 0, "medianOutputTokensPerSec": 98.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 98.7, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7256, "outputTokens": 2153, "cost": 0, "durationMs": 28132, "outputTokensPerSec": 98.74112408143682, "at": "2026-11-01T01:00:36.877Z" }, "lastUpdated": "2026-11-01T01:00:36.877Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:00:37.431Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45670, "meanTokensPerSec": 97.22, "genTokensPerSec": 97.22, "genTokens": 2655, "promptTokensPerSec": 1140.11, "peakRssMb": 21705, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7256, "output": 2153 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-symptom-debug", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "symptom-debug", "repetition": 3 }, "trialId": "symptom-debug-mlx-qwen3-8-27b-q4-2026-11-01T01-01-01-013Z-33md", "success": true, "includedInCapability": true, "durationMs": 47783, "timing": { "startedAt": "2026-11-01T01:01:01.014Z", "finishedAt": "2026-11-01T01:01:48.797Z", "timeToFirstArtifactMs": 43627, "timeToLastArtifactWriteMs": 43627, "timeToFirstTokenMs": 14328, "firstTurnTtftMs": 12668, "timeToFirstToolCallMs": 15854 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 18015, "peakCpuPercent": 118.3, "firstAtMs": 0, "lastAtMs": 45146 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 80052, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5434, "totalOutputTokens": 869, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:01:47.753Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45670, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 869, "promptTokensPerSec": null, "peakRssMb": 18015, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 673, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-symptom-debug", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "symptom-debug", "repetition": 3 }, "trialId": "symptom-debug-qwen3-8-27b-q4-2026-11-01T01-02-10-804Z-z2v7", "success": true, "includedInCapability": true, "durationMs": 43905, "timing": { "startedAt": "2026-11-01T01:02:10.805Z", "finishedAt": "2026-11-01T01:02:54.710Z", "timeToFirstArtifactMs": 37874, "timeToLastArtifactWriteMs": 37874, "timeToFirstTokenMs": 16396, "firstTurnTtftMs": 14571, "timeToFirstToolCallMs": 17461 }, "metrics": { "process": { "sampleCount": 9, "peakRssMb": 37183, "peakCpuPercent": 87.5, "firstAtMs": 0, "lastAtMs": 40134 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 83862, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5483, "totalOutputTokens": 581, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:02:52.513Z" } }, "source": "engine-log" }, "derived": { "durationMs": 40656, "meanTokensPerSec": 32, "genTokensPerSec": 32, "genTokens": 581, "promptTokensPerSec": 488.02, "peakRssMb": 37183, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-symptom-debug", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "symptom-debug", "repetition": 3 }, "trialId": "symptom-debug-mlx-gemma4-12b-q4-2026-11-01T01-03-16-564Z-5fxr", "success": true, "includedInCapability": true, "durationMs": 253670, "timing": { "startedAt": "2026-11-01T01:03:16.565Z", "finishedAt": "2026-11-01T01:07:30.235Z", "timeToFirstArtifactMs": 249622, "timeToLastArtifactWriteMs": 249622, "timeToFirstTokenMs": 11621, "firstTurnTtftMs": 9986, "timeToFirstToolCallMs": 14586 }, "metrics": { "process": { "sampleCount": 51, "peakRssMb": 10618, "peakCpuPercent": 121.6, "firstAtMs": 1, "lastAtMs": 250935 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 51, "peakUsedMb": 64189, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 21101, "totalOutputTokens": 8101, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 21085, "todayTokensCached": 4967, "todayTokensOut": 8303, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 21085, "totalTokensCached": 4967, "totalTokensOut": 8303, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14625, "cachedInputTokens": 0, "outputTokens": 111, "cost": 0, "durationMs": 18674, "outputTokensPerSec": 33.36867040824724, "at": "2026-11-01T01:07:26.196Z" }, "lastUpdated": "2026-11-01T01:07:26.196Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:07:29.106Z" } }, "source": "engine-log" }, "derived": { "durationMs": 251473, "meanTokensPerSec": 42, "genTokensPerSec": 42, "genTokens": 8101, "promptTokensPerSec": null, "peakRssMb": 10618, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21085, "output": 8303 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1162, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-core-data-wrangle", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "data-wrangle", "repetition": 3 }, "trialId": "data-wrangle-mlx-qwen3-8-27b-q4-2026-11-01T01-07-51-241Z-aqft", "success": true, "includedInCapability": true, "durationMs": 143226, "timing": { "startedAt": "2026-11-01T01:07:51.242Z", "finishedAt": "2026-11-01T01:10:14.468Z", "timeToFirstArtifactMs": 62253, "timeToLastArtifactWriteMs": 134215, "timeToFirstTokenMs": 15704, "firstTurnTtftMs": 14067, "timeToFirstToolCallMs": 18137 }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 17989, "peakCpuPercent": 102.2, "firstAtMs": 0, "lastAtMs": 140467 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 80565, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 9095, "totalOutputTokens": 2586, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:10:13.302Z" } }, "source": "engine-log" }, "derived": { "durationMs": 140990, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 2586, "promptTokensPerSec": null, "peakRssMb": 17989, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 516, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-data-wrangle", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "data-wrangle", "repetition": 3 }, "trialId": "data-wrangle-qwen3-8-27b-q4-2026-11-01T01-10-38-896Z-3eji", "success": true, "includedInCapability": true, "durationMs": 222030, "timing": { "startedAt": "2026-11-01T01:10:38.896Z", "finishedAt": "2026-11-01T01:14:20.926Z", "timeToFirstArtifactMs": 68029, "timeToLastArtifactWriteMs": 208768, "timeToFirstTokenMs": 18593, "firstTurnTtftMs": 16873, "timeToFirstToolCallMs": 21511 }, "metrics": { "process": { "sampleCount": 44, "peakRssMb": 38485, "peakCpuPercent": 124.8, "firstAtMs": 1, "lastAtMs": 215705 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 44, "peakUsedMb": 83830, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 14140, "totalOutputTokens": 5319, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:14:16.068Z" } }, "source": "engine-log" }, "derived": { "durationMs": 216227, "meanTokensPerSec": 32.12, "genTokensPerSec": 32.12, "genTokens": 5319, "promptTokensPerSec": 371.76, "peakRssMb": 38485, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-data-wrangle", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "data-wrangle", "repetition": 3 }, "trialId": "data-wrangle-mlx-gemma4-12b-q4-2026-11-01T01-14-44-501Z-5b5h", "success": true, "includedInCapability": true, "durationMs": 122996, "timing": { "startedAt": "2026-11-01T01:14:44.501Z", "finishedAt": "2026-11-01T01:16:47.497Z", "timeToFirstArtifactMs": 74340, "timeToLastArtifactWriteMs": 114314, "timeToFirstTokenMs": 12752, "firstTurnTtftMs": 11120, "timeToFirstToolCallMs": 32509 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 10437, "peakCpuPercent": 120.6, "firstAtMs": 0, "lastAtMs": 120472 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 63945, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 10709, "totalOutputTokens": 3345, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:16:46.568Z" } }, "source": "engine-log" }, "derived": { "durationMs": 120998, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 3345, "promptTokensPerSec": null, "peakRssMb": 10437, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1080, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-data-wrangle", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "data-wrangle", "repetition": 3 }, "trialId": "data-wrangle-gemma4-12b-q4-2026-11-01T01-17-09-570Z-bpkt", "success": true, "includedInCapability": true, "durationMs": 66845, "timing": { "startedAt": "2026-11-01T01:17:09.570Z", "finishedAt": "2026-11-01T01:18:16.415Z", "timeToFirstArtifactMs": 38031, "timeToLastArtifactWriteMs": 52940, "timeToFirstTokenMs": 10441, "firstTurnTtftMs": 8606, "timeToFirstToolCallMs": 17594 }, "metrics": { "process": { "sampleCount": 12, "peakRssMb": 19403, "peakCpuPercent": 120.4, "firstAtMs": 0, "lastAtMs": 55187 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 63318, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 10427, "totalOutputTokens": 3273, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:18:06.330Z" } }, "source": "engine-log" }, "derived": { "durationMs": 55709, "meanTokensPerSec": 87.61, "genTokensPerSec": 87.61, "genTokens": 3273, "promptTokensPerSec": 899.91, "peakRssMb": 19403, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-core-incident-postmortem", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "incident-postmortem", "repetition": 3 }, "trialId": "incident-postmortem-qwen3-8-27b-q4-2026-11-01T01-18-39-026Z-xdsu", "success": true, "includedInCapability": true, "durationMs": 410911, "timing": { "startedAt": "2026-11-01T01:18:39.026Z", "finishedAt": "2026-11-01T01:25:29.937Z", "timeToFirstArtifactMs": 300480, "timeToLastArtifactWriteMs": 404664, "timeToFirstTokenMs": 32193, "firstTurnTtftMs": 30384, "timeToFirstToolCallMs": 39583 }, "metrics": { "process": { "sampleCount": 82, "peakRssMb": 40546, "peakCpuPercent": 118.7, "firstAtMs": 0, "lastAtMs": 406337 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 82, "peakUsedMb": 86472, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 35445, "totalOutputTokens": 7445, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 35456, "todayTokensCached": 31531, "todayTokensOut": 690, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 35456, "totalTokensCached": 31531, "totalTokensOut": 690, "totalCost": 0, "medianOutputTokensPerSec": 28.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.6, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 15502, "cachedInputTokens": 11816, "outputTokens": 251, "cost": 0, "durationMs": 52087, "outputTokensPerSec": 34.579973658359265, "at": "2026-11-01T01:25:23.693Z" }, "lastUpdated": "2026-11-01T01:25:23.693Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:25:26.954Z" } }, "source": "engine-log" }, "derived": { "durationMs": 406881, "meanTokensPerSec": 24.62, "genTokensPerSec": 24.62, "genTokens": 7445, "promptTokensPerSec": 382.03, "peakRssMb": 40546, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35456, "output": 690 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-incident-postmortem", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "incident-postmortem", "repetition": 3 }, "trialId": "incident-postmortem-mlx-gemma4-12b-q4-2026-11-01T01-25-52-129Z-1hb1", "success": true, "includedInCapability": true, "durationMs": 354036, "timing": { "startedAt": "2026-11-01T01:25:52.130Z", "finishedAt": "2026-11-01T01:31:46.166Z", "timeToFirstArtifactMs": 230880, "timeToLastArtifactWriteMs": 351750, "timeToFirstTokenMs": 17850, "firstTurnTtftMs": 16215, "timeToFirstToolCallMs": 39148 }, "metrics": { "process": { "sampleCount": 71, "peakRssMb": 10391, "peakCpuPercent": 100.6, "firstAtMs": 1, "lastAtMs": 351306 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 71, "peakUsedMb": 64329, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 46633, "totalOutputTokens": 9773, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 46521, "todayTokensCached": 44520, "todayTokensOut": 491, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 46521, "totalTokensCached": 44520, "totalTokensOut": 491, "totalCost": 0, "medianOutputTokensPerSec": 40.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.3, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15264, "cachedInputTokens": 14695, "outputTokens": 32, "cost": 0, "durationMs": 78440, "outputTokensPerSec": 40.30712297322386, "at": "2026-11-01T01:31:43.880Z" }, "lastUpdated": "2026-11-01T01:31:43.880Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:31:45.024Z" } }, "source": "engine-log" }, "derived": { "durationMs": 351828, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 9773, "promptTokensPerSec": null, "peakRssMb": 10391, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 46521, "output": 491 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1016, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-incident-postmortem", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "incident-postmortem", "repetition": 3 }, "trialId": "incident-postmortem-gemma4-12b-q4-2026-11-01T01-32-09-766Z-rupe", "success": true, "includedInCapability": true, "durationMs": 159309, "timing": { "startedAt": "2026-11-01T01:32:09.766Z", "finishedAt": "2026-11-01T01:34:49.075Z", "timeToFirstArtifactMs": 86786, "timeToLastArtifactWriteMs": 153598, "timeToFirstTokenMs": 15915, "firstTurnTtftMs": 14054, "timeToFirstToolCallMs": 26981 }, "metrics": { "process": { "sampleCount": 32, "peakRssMb": 25717, "peakCpuPercent": 111.6, "firstAtMs": 0, "lastAtMs": 155517 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 69542, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 40562, "totalOutputTokens": 5808, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 29181, "todayTokensCached": 16918, "todayTokensOut": 2810, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 29181, "totalTokensCached": 16918, "totalTokensOut": 2810, "totalCost": 0, "medianOutputTokensPerSec": 57.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 57.7, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11153, "outputTokens": 2647, "cost": 0, "durationMs": 61625, "outputTokensPerSec": 52.73088174723119, "at": "2026-11-01T01:34:43.364Z" }, "lastUpdated": "2026-11-01T01:34:43.364Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:34:46.874Z" } }, "source": "engine-log" }, "derived": { "durationMs": 156059, "meanTokensPerSec": 56.29, "genTokensPerSec": 56.29, "genTokens": 5808, "promptTokensPerSec": 949.21, "peakRssMb": 25717, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 29181, "output": 2810 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-incident-postmortem", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "incident-postmortem", "repetition": 3 }, "trialId": "incident-postmortem-mlx-qwen3-8-27b-q4-2026-11-01T01-35-14-261Z-7yx3", "success": true, "includedInCapability": true, "durationMs": 439614, "timing": { "startedAt": "2026-11-01T01:35:14.262Z", "finishedAt": "2026-11-01T01:42:33.876Z", "timeToFirstArtifactMs": 437233, "timeToLastArtifactWriteMs": 437233, "timeToFirstTokenMs": 28618, "firstTurnTtftMs": 26981, "timeToFirstToolCallMs": 37700 }, "metrics": { "process": { "sampleCount": 88, "peakRssMb": 18004, "peakCpuPercent": 115.6, "firstAtMs": 0, "lastAtMs": 436447 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 88, "peakUsedMb": 80310, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17813, "totalOutputTokens": 9409, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:42:32.296Z" } }, "source": "engine-log" }, "derived": { "durationMs": 436971, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 9409, "promptTokensPerSec": null, "peakRssMb": 18004, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 553, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-core-ops-runbook-anomaly", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 3 }, "trialId": "ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-11-01T01-42-55-909Z-v8kh", "success": true, "includedInCapability": true, "durationMs": 103102, "timing": { "startedAt": "2026-11-01T01:42:55.910Z", "finishedAt": "2026-11-01T01:44:39.012Z", "timeToFirstArtifactMs": 30196, "timeToLastArtifactWriteMs": 98139, "timeToFirstTokenMs": 17556, "firstTurnTtftMs": 16026, "timeToFirstToolCallMs": 27646 }, "metrics": { "process": { "sampleCount": 21, "peakRssMb": 10646, "peakCpuPercent": 118.1, "firstAtMs": 0, "lastAtMs": 100392 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 64514, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47720, "totalOutputTokens": 369, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 27071, "todayTokensCached": 18370, "todayTokensOut": 317, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 27071, "totalTokensCached": 18370, "totalTokensOut": 317, "totalCost": 0, "medianOutputTokensPerSec": 37, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8401, "cachedInputTokens": 0, "outputTokens": 121, "cost": 0, "durationMs": 31768, "outputTokensPerSec": 40.15800781386254, "at": "2026-11-01T01:44:34.048Z" }, "lastUpdated": "2026-11-01T01:44:34.048Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:44:37.806Z" } }, "source": "engine-log" }, "derived": { "durationMs": 100937, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 369, "promptTokensPerSec": null, "peakRssMb": 10646, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 27071, "output": 317 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1179, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-ops-runbook-anomaly", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 3 }, "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-11-01T01-45-00-961Z-bao3", "success": true, "includedInCapability": true, "durationMs": 94110, "timing": { "startedAt": "2026-11-01T01:45:00.962Z", "finishedAt": "2026-11-01T01:46:35.072Z", "timeToFirstArtifactMs": 20801, "timeToLastArtifactWriteMs": 90756, "timeToFirstTokenMs": 14982, "firstTurnTtftMs": 13127, "timeToFirstToolCallMs": 19000 }, "metrics": { "process": { "sampleCount": 19, "peakRssMb": 23167, "peakCpuPercent": 106.5, "firstAtMs": 0, "lastAtMs": 90325 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 19, "peakUsedMb": 67305, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 56031, "totalOutputTokens": 1924, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 60713, "todayTokensCached": 44625, "todayTokensOut": 445, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 60713, "totalTokensCached": 44625, "totalTokensOut": 445, "totalCost": 0, "medianOutputTokensPerSec": 88.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88.5, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8454, "outputTokens": 107, "cost": 0, "durationMs": 24297, "outputTokensPerSec": 85.55334856613396, "at": "2026-11-01T01:46:31.718Z" }, "lastUpdated": "2026-11-01T01:46:31.718Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:46:32.876Z" } }, "source": "engine-log" }, "derived": { "durationMs": 90869, "meanTokensPerSec": 77.09, "genTokensPerSec": 77.09, "genTokens": 1924, "promptTokensPerSec": 1041.96, "peakRssMb": 23167, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 60713, "output": 445 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-ops-runbook-anomaly", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 3 }, "trialId": "ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-11-01T01-46-56-510Z-4f6u", "success": true, "includedInCapability": true, "durationMs": 88154, "timing": { "startedAt": "2026-11-01T01:46:56.511Z", "finishedAt": "2026-11-01T01:48:24.665Z", "timeToFirstArtifactMs": 70944, "timeToLastArtifactWriteMs": 83888, "timeToFirstTokenMs": 27509, "firstTurnTtftMs": 25862, "timeToFirstToolCallMs": 32082 }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 17970, "peakCpuPercent": 120.9, "firstAtMs": 0, "lastAtMs": 85280 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 80997, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12296, "totalOutputTokens": 1356, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12232, "todayTokensCached": 12183, "todayTokensOut": 371, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12232, "totalTokensCached": 12183, "totalTokensOut": 371, "totalCost": 0, "medianOutputTokensPerSec": 29.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.5, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12232, "cachedInputTokens": 12183, "outputTokens": 371, "cost": 0, "durationMs": 82241, "outputTokensPerSec": 29.5025531659363, "at": "2026-11-01T01:48:20.399Z" }, "lastUpdated": "2026-11-01T01:48:20.399Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:48:23.400Z" } }, "source": "engine-log" }, "derived": { "durationMs": 85821, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 1356, "promptTokensPerSec": null, "peakRssMb": 17970, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12232, "output": 371 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 587, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-ops-runbook-anomaly", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 3 }, "trialId": "ops-runbook-anomaly-qwen3-8-27b-q4-2026-11-01T01-48-46-428Z-cw9k", "success": true, "includedInCapability": true, "durationMs": 224386, "timing": { "startedAt": "2026-11-01T01:48:46.429Z", "finishedAt": "2026-11-01T01:52:30.815Z", "timeToFirstArtifactMs": 84731, "timeToLastArtifactWriteMs": 218550, "timeToFirstTokenMs": 32202, "firstTurnTtftMs": 30383, "timeToFirstToolCallMs": 33028 }, "metrics": { "process": { "sampleCount": 45, "peakRssMb": 44014, "peakCpuPercent": 117.6, "firstAtMs": 0, "lastAtMs": 220739 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 45, "peakUsedMb": 88823, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 42995, "totalOutputTokens": 3231, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 21569, "todayTokensCached": 20869, "todayTokensOut": 393, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 21569, "totalTokensCached": 20869, "totalTokensOut": 393, "totalCost": 0, "medianOutputTokensPerSec": 26, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8619, "cachedInputTokens": 8285, "outputTokens": 93, "cost": 0, "durationMs": 96431, "outputTokensPerSec": 23.189019495159926, "at": "2026-11-01T01:52:24.979Z" }, "lastUpdated": "2026-11-01T01:52:24.979Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:52:28.754Z" } }, "source": "engine-log" }, "derived": { "durationMs": 221280, "meanTokensPerSec": 31.19, "genTokensPerSec": 31.19, "genTokens": 3231, "promptTokensPerSec": 425.41, "peakRssMb": 44014, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 21569, "output": 393 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-core-plan-and-estimate", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-gemma4-12b-q4-2026-11-01T01-52-53-207Z-rwvy", "success": true, "includedInCapability": true, "durationMs": 34778, "timing": { "startedAt": "2026-11-01T01:52:53.208Z", "finishedAt": "2026-11-01T01:53:27.986Z", "timeToFirstArtifactMs": 26813, "timeToLastArtifactWriteMs": 28257, "timeToFirstTokenMs": 10994, "firstTurnTtftMs": 9167, "timeToFirstToolCallMs": 16305 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 14877, "peakCpuPercent": 121.4, "firstAtMs": 0, "lastAtMs": 30099 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 58646, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6209, "totalOutputTokens": 985, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6204, "todayTokensCached": 5649, "todayTokensOut": 34, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6204, "totalTokensCached": 5649, "totalTokensOut": 34, "totalCost": 0, "medianOutputTokensPerSec": 64.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 64.5, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6204, "cachedInputTokens": 5649, "outputTokens": 34, "cost": 0, "durationMs": 26430, "outputTokensPerSec": 64.4664715741931, "at": "2026-11-01T01:53:21.465Z" }, "lastUpdated": "2026-11-01T01:53:21.465Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:53:24.895Z" } }, "source": "engine-log" }, "derived": { "durationMs": 30640, "meanTokensPerSec": 63.39, "genTokensPerSec": 63.39, "genTokens": 985, "promptTokensPerSec": 864.05, "peakRssMb": 14877, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6204, "output": 34 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-core-plan-and-estimate", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T01-53-48-630Z-lhjf", "success": true, "includedInCapability": true, "durationMs": 67603, "timing": { "startedAt": "2026-11-01T01:53:48.632Z", "finishedAt": "2026-11-01T01:54:56.235Z", "timeToFirstArtifactMs": 63199, "timeToLastArtifactWriteMs": 63199, "timeToFirstTokenMs": 16272, "firstTurnTtftMs": 14631, "timeToFirstToolCallMs": 17999 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 17980, "peakCpuPercent": 113.6, "firstAtMs": 0, "lastAtMs": 65215 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 79434, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4740, "totalOutputTokens": 1167, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:54:55.438Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65737, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 1167, "promptTokensPerSec": null, "peakRssMb": 17980, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 552, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-plan-and-estimate", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-qwen3-8-27b-q4-2026-11-01T01-55-43-688Z-g0gc", "success": true, "includedInCapability": true, "durationMs": 68651, "timing": { "startedAt": "2026-11-01T01:55:43.689Z", "finishedAt": "2026-11-01T01:56:52.340Z", "timeToFirstArtifactMs": 62418, "timeToLastArtifactWriteMs": 62418, "timeToFirstTokenMs": 17493, "firstTurnTtftMs": 15540, "timeToFirstToolCallMs": 18423 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 37052, "peakCpuPercent": 114.2, "firstAtMs": 0, "lastAtMs": 65213 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 83347, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 4809, "totalOutputTokens": 1120, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:56:50.668Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65733, "meanTokensPerSec": 25.68, "genTokensPerSec": 25.68, "genTokens": 1120, "promptTokensPerSec": 471.82, "peakRssMb": 37052, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-plan-and-estimate", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T01-57-17-006Z-4xa8", "success": true, "includedInCapability": true, "durationMs": 37436, "timing": { "startedAt": "2026-11-01T01:57:17.007Z", "finishedAt": "2026-11-01T01:57:54.443Z", "timeToFirstArtifactMs": 33940, "timeToLastArtifactWriteMs": 35837, "timeToFirstTokenMs": 11907, "firstTurnTtftMs": 10274, "timeToFirstToolCallMs": 22209 }, "metrics": { "process": { "sampleCount": 8, "peakRssMb": 10434, "peakCpuPercent": 102.2, "firstAtMs": 0, "lastAtMs": 35122 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 63738, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6178, "totalOutputTokens": 871, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6146, "todayTokensCached": 5631, "todayTokensOut": 57, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6146, "totalTokensCached": 5631, "totalTokensOut": 57, "totalCost": 0, "medianOutputTokensPerSec": 45.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 45.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6146, "cachedInputTokens": 5631, "outputTokens": 57, "cost": 0, "durationMs": 34204, "outputTokensPerSec": 45.41568067658686, "at": "2026-11-01T01:57:52.844Z" }, "lastUpdated": "2026-11-01T01:57:52.844Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T01:57:53.732Z" } }, "source": "engine-log" }, "derived": { "durationMs": 35661, "meanTokensPerSec": 44, "genTokensPerSec": 44, "genTokens": 871, "promptTokensPerSec": null, "peakRssMb": 10434, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 6146, "output": 57 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1319, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-core-conflict-synthesis", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "core", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T01-58-17-534Z-e91g", "success": true, "includedInCapability": true, "durationMs": 183366, "timing": { "startedAt": "2026-11-01T01:58:17.534Z", "finishedAt": "2026-11-01T02:01:20.900Z", "timeToFirstArtifactMs": 88387, "timeToLastArtifactWriteMs": 178227, "timeToFirstTokenMs": 25229, "firstTurnTtftMs": 23540, "timeToFirstToolCallMs": 29704 }, "metrics": { "process": { "sampleCount": 37, "peakRssMb": 17978, "peakCpuPercent": 120.4, "firstAtMs": 0, "lastAtMs": 180610 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 37, "peakUsedMb": 79843, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 23170, "totalOutputTokens": 3952, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18296, "todayTokensCached": 11668, "todayTokensOut": 2189, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18296, "totalTokensCached": 11668, "totalTokensOut": 2189, "totalCost": 0, "medianOutputTokensPerSec": 34.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 34.7, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5609, "cachedInputTokens": 0, "outputTokens": 1072, "cost": 0, "durationMs": 51088, "outputTokensPerSec": 37.00867879777663, "at": "2026-11-01T02:01:15.763Z" }, "lastUpdated": "2026-11-01T02:01:15.763Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:01:19.753Z" } }, "source": "engine-log" }, "derived": { "durationMs": 181153, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 3952, "promptTokensPerSec": null, "peakRssMb": 17978, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 18296, "output": 2189 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 614.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-core-conflict-synthesis", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "core", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-qwen3-8-27b-q4-2026-11-01T02-01-42-847Z-m435", "success": true, "includedInCapability": true, "durationMs": 731826, "timing": { "startedAt": "2026-11-01T02:01:42.848Z", "finishedAt": "2026-11-01T02:13:54.674Z", "timeToFirstArtifactMs": 180817, "timeToLastArtifactWriteMs": 725039, "timeToFirstTokenMs": 34856, "firstTurnTtftMs": 33031, "timeToFirstToolCallMs": 39430 }, "metrics": { "process": { "sampleCount": 146, "peakRssMb": 49049, "peakCpuPercent": 118.8, "firstAtMs": 0, "lastAtMs": 727460 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 146, "peakUsedMb": 93966, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 172769, "totalOutputTokens": 7996, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 67188, "todayTokensCached": 27152, "todayTokensOut": 3490, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 67188, "totalTokensCached": 27152, "totalTokensOut": 3490, "totalCost": 0, "medianOutputTokensPerSec": 36.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.5, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 15759, "outputTokens": 236, "cost": 0, "durationMs": 141962, "outputTokensPerSec": 37.00846848248586, "at": "2026-11-01T02:13:47.890Z" }, "lastUpdated": "2026-11-01T02:13:47.890Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:13:51.899Z" } }, "source": "engine-log" }, "derived": { "durationMs": 728001, "meanTokensPerSec": 32.18, "genTokensPerSec": 32.18, "genTokens": 7996, "promptTokensPerSec": 382.4, "peakRssMb": 49049, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 67188, "output": 3490 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-core-conflict-synthesis", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "core", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T02-14-16-657Z-7y28", "success": true, "includedInCapability": true, "durationMs": 123344, "timing": { "startedAt": "2026-11-01T02:14:16.658Z", "finishedAt": "2026-11-01T02:16:20.002Z", "timeToFirstArtifactMs": 41191, "timeToLastArtifactWriteMs": 116650, "timeToFirstTokenMs": 17197, "firstTurnTtftMs": 15553, "timeToFirstToolCallMs": 29616 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 10663, "peakCpuPercent": 119.9, "firstAtMs": 0, "lastAtMs": 120457 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 64243, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47884, "totalOutputTokens": 1143, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 27285, "todayTokensCached": 19001, "todayTokensOut": 603, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 27285, "totalTokensCached": 19001, "totalTokensOut": 603, "totalCost": 0, "medianOutputTokensPerSec": 38.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.4, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8841, "cachedInputTokens": 1178, "outputTokens": 416, "cost": 0, "durationMs": 30131, "outputTokensPerSec": 38.44490661805966, "at": "2026-11-01T02:16:13.311Z" }, "lastUpdated": "2026-11-01T02:16:13.311Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:16:18.723Z" } }, "source": "engine-log" }, "derived": { "durationMs": 121000, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 1143, "promptTokensPerSec": null, "peakRssMb": 10663, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 27285, "output": 603 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1180.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-core-conflict-synthesis", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "core", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-gemma4-12b-q4-2026-11-01T02-16-43-309Z-zf7z", "success": true, "includedInCapability": true, "durationMs": 34998, "timing": { "startedAt": "2026-11-01T02:16:43.309Z", "finishedAt": "2026-11-01T02:17:18.307Z", "timeToFirstArtifactMs": 28518, "timeToLastArtifactWriteMs": 30366, "timeToFirstTokenMs": 14989, "firstTurnTtftMs": 13019, "timeToFirstToolCallMs": 21144 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 15003, "peakCpuPercent": 96.5, "firstAtMs": 0, "lastAtMs": 30110 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 58852, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 11115, "totalOutputTokens": 969, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11112, "todayTokensCached": 10655, "todayTokensOut": 88, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11112, "totalTokensCached": 10655, "totalTokensOut": 88, "totalCost": 0, "medianOutputTokensPerSec": 73.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 73.6, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11112, "cachedInputTokens": 10655, "outputTokens": 88, "cost": 0, "durationMs": 28396, "outputTokensPerSec": 73.5696412761204, "at": "2026-11-01T02:17:13.675Z" }, "lastUpdated": "2026-11-01T02:17:13.675Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:17:15.112Z" } }, "source": "engine-log" }, "derived": { "durationMs": 30652, "meanTokensPerSec": 70.71, "genTokensPerSec": 70.71, "genTokens": 969, "promptTokensPerSec": 1020.09, "peakRssMb": 15003, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11112, "output": 88 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-productivity-constrained-comms", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "constrained-comms", "repetition": 3 }, "trialId": "constrained-comms-qwen3-8-27b-q4-2026-11-01T02-17-41-717Z-mi9h", "success": true, "includedInCapability": true, "durationMs": 68584, "timing": { "startedAt": "2026-11-01T02:17:41.717Z", "finishedAt": "2026-11-01T02:18:50.301Z", "timeToFirstArtifactMs": 40372, "timeToLastArtifactWriteMs": 66413, "timeToFirstTokenMs": 28843, "firstTurnTtftMs": 26935, "timeToFirstToolCallMs": 29636 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 39931, "peakCpuPercent": 112.2, "firstAtMs": 0, "lastAtMs": 65227 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 85173, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 21661, "totalOutputTokens": 616, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 15856, "todayTokensCached": 11878, "todayTokensOut": 504, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 15856, "totalTokensCached": 11878, "totalTokensOut": 504, "totalCost": 0, "medianOutputTokensPerSec": 43.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 43.4, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 6727, "cachedInputTokens": 6464, "outputTokens": 102, "cost": 0, "durationMs": 4923, "outputTokensPerSec": 51.72974537185236, "at": "2026-11-01T02:18:48.133Z" }, "lastUpdated": "2026-11-01T02:18:48.133Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:18:48.617Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65753, "meanTokensPerSec": 46.22, "genTokensPerSec": 46.22, "genTokens": 616, "promptTokensPerSec": 544.26, "peakRssMb": 39931, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15856, "output": 504 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-constrained-comms", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "constrained-comms", "repetition": 3 }, "trialId": "constrained-comms-mlx-gemma4-12b-q4-2026-11-01T02-19-15-016Z-38lk", "success": true, "includedInCapability": true, "durationMs": 52481, "timing": { "startedAt": "2026-11-01T02:19:15.016Z", "finishedAt": "2026-11-01T02:20:07.497Z", "timeToFirstArtifactMs": 31519, "timeToLastArtifactWriteMs": 50634, "timeToFirstTokenMs": 15539, "firstTurnTtftMs": 13913, "timeToFirstToolCallMs": 22915 }, "metrics": { "process": { "sampleCount": 11, "peakRssMb": 10482, "peakCpuPercent": 116.6, "firstAtMs": 0, "lastAtMs": 50183 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 11, "peakUsedMb": 64008, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 26759, "totalOutputTokens": 440, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 11064, "todayTokensCached": 0, "todayTokensOut": 432, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 11064, "totalTokensCached": 0, "totalTokensOut": 432, "totalCost": 0, "medianOutputTokensPerSec": 44.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 44.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5505, "cachedInputTokens": 0, "outputTokens": 260, "cost": 0, "durationMs": 13793, "outputTokensPerSec": 43.719101228924124, "at": "2026-11-01T02:20:05.654Z" }, "lastUpdated": "2026-11-01T02:20:05.654Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:20:06.805Z" } }, "source": "engine-log" }, "derived": { "durationMs": 50727, "meanTokensPerSec": 44, "genTokensPerSec": 44, "genTokens": 440, "promptTokensPerSec": null, "peakRssMb": 10482, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11064, "output": 432 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1584, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-constrained-comms", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "constrained-comms", "repetition": 3 }, "trialId": "constrained-comms-gemma4-12b-q4-2026-11-01T02-20-32-103Z-ghjq", "success": true, "includedInCapability": true, "durationMs": 27745, "timing": { "startedAt": "2026-11-01T02:20:32.104Z", "finishedAt": "2026-11-01T02:20:59.849Z", "timeToFirstArtifactMs": 25203, "timeToLastArtifactWriteMs": 25205, "timeToFirstTokenMs": 14127, "firstTurnTtftMs": 12390, "timeToFirstToolCallMs": 17153 }, "metrics": { "process": { "sampleCount": 6, "peakRssMb": 15156, "peakCpuPercent": 121.7, "firstAtMs": 1, "lastAtMs": 25084 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 59371, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15528, "totalOutputTokens": 520, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5605, "todayTokensCached": 0, "todayTokensOut": 229, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5605, "totalTokensCached": 0, "totalTokensOut": 229, "totalCost": 0, "medianOutputTokensPerSec": 80, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 80, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5605, "outputTokens": 229, "cost": 0, "durationMs": 23468, "outputTokensPerSec": 80.0097976910155, "at": "2026-11-01T02:20:57.309Z" }, "lastUpdated": "2026-11-01T02:20:57.309Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:20:58.664Z" } }, "source": "engine-log" }, "derived": { "durationMs": 25616, "meanTokensPerSec": 88.21, "genTokensPerSec": 88.21, "genTokens": 520, "promptTokensPerSec": 1126.07, "peakRssMb": 15156, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5605, "output": 229 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-constrained-comms", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "constrained-comms", "repetition": 3 }, "trialId": "constrained-comms-mlx-qwen3-8-27b-q4-2026-11-01T02-21-20-714Z-y3cq", "success": true, "includedInCapability": true, "durationMs": 78055, "timing": { "startedAt": "2026-11-01T02:21:20.714Z", "finishedAt": "2026-11-01T02:22:38.769Z", "timeToFirstArtifactMs": 36530, "timeToLastArtifactWriteMs": 73109, "timeToFirstTokenMs": 25238, "firstTurnTtftMs": 23599, "timeToFirstToolCallMs": 26410 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 18203, "peakCpuPercent": 112.1, "firstAtMs": 0, "lastAtMs": 75275 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 82220, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 28877, "totalOutputTokens": 6, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 11656, "todayTokensCached": 1390, "todayTokensOut": 487, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 11656, "totalTokensCached": 1390, "totalTokensOut": 487, "totalCost": 0, "medianOutputTokensPerSec": 45.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 45.6, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 4536, "cachedInputTokens": 1390, "outputTokens": 109, "cost": 0, "durationMs": 11702, "outputTokensPerSec": 45.55104828248733, "at": "2026-11-01T02:22:33.825Z" }, "lastUpdated": "2026-11-01T02:22:33.825Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:22:37.601Z" } }, "source": "engine-log" }, "derived": { "durationMs": 75820, "meanTokensPerSec": 380.94, "genTokensPerSec": null, "genTokens": 6, "promptTokensPerSec": null, "peakRssMb": 18203, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11656, "output": 487 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 798, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-productivity-records-intake", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "records-intake", "repetition": 3 }, "trialId": "records-intake-mlx-gemma4-12b-q4-2026-11-01T02-23-00-672Z-jeky", "success": true, "includedInCapability": true, "durationMs": 57819, "timing": { "startedAt": "2026-11-01T02:23:00.673Z", "finishedAt": "2026-11-01T02:23:58.492Z", "timeToFirstArtifactMs": 53757, "timeToLastArtifactWriteMs": 53757, "timeToFirstTokenMs": 11745, "firstTurnTtftMs": 10120, "timeToFirstToolCallMs": 37286 }, "metrics": { "process": { "sampleCount": 12, "peakRssMb": 10405, "peakCpuPercent": 100.8, "firstAtMs": 0, "lastAtMs": 55195 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 64067, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6412, "totalOutputTokens": 1528, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:23:57.458Z" } }, "source": "engine-log" }, "derived": { "durationMs": 55719, "meanTokensPerSec": 44, "genTokensPerSec": 44, "genTokens": 1528, "promptTokensPerSec": null, "peakRssMb": 10405, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1333, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-records-intake", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "records-intake", "repetition": 3 }, "trialId": "records-intake-gemma4-12b-q4-2026-11-01T02-24-20-268Z-o30q", "success": true, "includedInCapability": true, "durationMs": 125347, "timing": { "startedAt": "2026-11-01T02:24:20.268Z", "finishedAt": "2026-11-01T02:26:25.615Z", "timeToFirstArtifactMs": 40454, "timeToLastArtifactWriteMs": 119311, "timeToFirstTokenMs": 9903, "firstTurnTtftMs": 8053, "timeToFirstToolCallMs": 19062 }, "metrics": { "process": { "sampleCount": 25, "peakRssMb": 26630, "peakCpuPercent": 98.4, "firstAtMs": 0, "lastAtMs": 120543 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 70556, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15246, "totalOutputTokens": 5310, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15210, "todayTokensCached": 13355, "todayTokensOut": 200, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15210, "totalTokensCached": 13355, "totalTokensOut": 200, "totalCost": 0, "medianOutputTokensPerSec": 67.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 67.2, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7595, "cachedInputTokens": 7029, "outputTokens": 47, "cost": 0, "durationMs": 32201, "outputTokensPerSec": 61.5392848064857, "at": "2026-11-01T02:26:19.579Z" }, "lastUpdated": "2026-11-01T02:26:19.579Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:26:22.405Z" } }, "source": "engine-log" }, "derived": { "durationMs": 121086, "meanTokensPerSec": 96.03, "genTokensPerSec": 96.03, "genTokens": 5310, "promptTokensPerSec": 1039.96, "peakRssMb": 26630, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15210, "output": 200 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-records-intake", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "records-intake", "repetition": 3 }, "trialId": "records-intake-mlx-qwen3-8-27b-q4-2026-11-01T02-26-47-094Z-myp0", "success": true, "includedInCapability": true, "durationMs": 67716, "timing": { "startedAt": "2026-11-01T02:26:47.095Z", "finishedAt": "2026-11-01T02:27:54.811Z", "timeToFirstArtifactMs": 62900, "timeToLastArtifactWriteMs": 62900, "timeToFirstTokenMs": 14556, "firstTurnTtftMs": 13029, "timeToFirstToolCallMs": 15351 }, "metrics": { "process": { "sampleCount": 14, "peakRssMb": 18015, "peakCpuPercent": 117.5, "firstAtMs": 0, "lastAtMs": 65215 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 14, "peakUsedMb": 80364, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5602, "totalOutputTokens": 1921, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:27:53.795Z" } }, "source": "engine-log" }, "derived": { "durationMs": 65736, "meanTokensPerSec": 48, "genTokensPerSec": 48, "genTokens": 1921, "promptTokensPerSec": null, "peakRssMb": 18015, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 699, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-records-intake", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "records-intake", "repetition": 3 }, "trialId": "records-intake-qwen3-8-27b-q4-2026-11-01T02-28-16-776Z-wnqa", "success": true, "includedInCapability": true, "durationMs": 417884, "timing": { "startedAt": "2026-11-01T02:28:16.778Z", "finishedAt": "2026-11-01T02:35:14.662Z", "timeToFirstArtifactMs": 218506, "timeToLastArtifactWriteMs": 409545, "timeToFirstTokenMs": 16878, "firstTurnTtftMs": 15081, "timeToFirstToolCallMs": 18604 }, "metrics": { "process": { "sampleCount": 83, "peakRssMb": 40688, "peakCpuPercent": 101.8, "firstAtMs": 0, "lastAtMs": 411362 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 83, "peakUsedMb": 87061, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 9226, "totalOutputTokens": 11784, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:35:09.710Z" } }, "source": "engine-log" }, "derived": { "durationMs": 411884, "meanTokensPerSec": 31.32, "genTokensPerSec": 31.32, "genTokens": 11784, "promptTokensPerSec": 377.87, "peakRssMb": 40688, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-productivity-plan-and-estimate", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-gemma4-12b-q4-2026-11-01T02-35-36-978Z-fuhe", "success": true, "includedInCapability": true, "durationMs": 35257, "timing": { "startedAt": "2026-11-01T02:35:36.978Z", "finishedAt": "2026-11-01T02:36:12.235Z", "timeToFirstArtifactMs": 29505, "timeToLastArtifactWriteMs": 29505, "timeToFirstTokenMs": 11275, "firstTurnTtftMs": 9480, "timeToFirstToolCallMs": 16855 }, "metrics": { "process": { "sampleCount": 7, "peakRssMb": 14921, "peakCpuPercent": 105.6, "firstAtMs": 0, "lastAtMs": 30102 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 59462, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 5654, "totalOutputTokens": 1018, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:36:08.638Z" } }, "source": "engine-log" }, "derived": { "durationMs": 30622, "meanTokensPerSec": 58.35, "genTokensPerSec": 58.35, "genTokens": 1018, "promptTokensPerSec": 858.18, "peakRssMb": 14921, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-plan-and-estimate", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T02-36-36-540Z-1pas", "success": true, "includedInCapability": true, "durationMs": 153225, "timing": { "startedAt": "2026-11-01T02:36:36.541Z", "finishedAt": "2026-11-01T02:39:09.766Z", "timeToFirstArtifactMs": 92687, "timeToLastArtifactWriteMs": 149317, "timeToFirstTokenMs": 16423, "firstTurnTtftMs": 14769, "timeToFirstToolCallMs": 18440 }, "metrics": { "process": { "sampleCount": 31, "peakRssMb": 18009, "peakCpuPercent": 122.5, "firstAtMs": 0, "lastAtMs": 150491 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 80085, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15686, "totalOutputTokens": 2872, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 10530, "todayTokensCached": 4956, "todayTokensOut": 1135, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 10530, "totalTokensCached": 4956, "totalTokensOut": 1135, "totalCost": 0, "medianOutputTokensPerSec": 33.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.3, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5536, "cachedInputTokens": 0, "outputTokens": 637, "cost": 0, "durationMs": 37234, "outputTokensPerSec": 37.501936733846655, "at": "2026-11-01T02:39:05.863Z" }, "lastUpdated": "2026-11-01T02:39:05.863Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:39:08.626Z" } }, "source": "engine-log" }, "derived": { "durationMs": 151026, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 2872, "promptTokensPerSec": null, "peakRssMb": 18009, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 10530, "output": 1135 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 557, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-plan-and-estimate", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-qwen3-8-27b-q4-2026-11-01T02-39-34-132Z-cz68", "success": true, "includedInCapability": true, "durationMs": 129944, "timing": { "startedAt": "2026-11-01T02:39:34.133Z", "finishedAt": "2026-11-01T02:41:44.077Z", "timeToFirstArtifactMs": 79082, "timeToLastArtifactWriteMs": 122794, "timeToFirstTokenMs": 18756, "firstTurnTtftMs": 16930, "timeToFirstToolCallMs": 19993 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 37544, "peakCpuPercent": 109.6, "firstAtMs": 0, "lastAtMs": 125400 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 83935, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6282, "totalOutputTokens": 2517, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:41:41.092Z" } }, "source": "engine-log" }, "derived": { "durationMs": 125920, "meanTokensPerSec": 25.62, "genTokensPerSec": 25.62, "genTokens": 2517, "promptTokensPerSec": 392.83, "peakRssMb": 37544, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-plan-and-estimate", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 3 }, "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T02-42-06-414Z-46bn", "success": true, "includedInCapability": true, "durationMs": 47734, "timing": { "startedAt": "2026-11-01T02:42:06.414Z", "finishedAt": "2026-11-01T02:42:54.148Z", "timeToFirstArtifactMs": 42684, "timeToLastArtifactWriteMs": 45956, "timeToFirstTokenMs": 12781, "firstTurnTtftMs": 11141, "timeToFirstToolCallMs": 26697 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 10384, "peakCpuPercent": 99.8, "firstAtMs": 0, "lastAtMs": 45171 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 64895, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 6286, "totalOutputTokens": 740, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6238, "todayTokensCached": 5680, "todayTokensOut": 97, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6238, "totalTokensCached": 5680, "totalTokensOut": 97, "totalCost": 0, "medianOutputTokensPerSec": 40.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.3, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6238, "cachedInputTokens": 5680, "outputTokens": 97, "cost": 0, "durationMs": 44316, "outputTokensPerSec": 40.33734061925665, "at": "2026-11-01T02:42:52.370Z" }, "lastUpdated": "2026-11-01T02:42:52.370Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:42:53.196Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45715, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 740, "promptTokensPerSec": null, "peakRssMb": 10384, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 6238, "output": 97 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1093, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-productivity-meeting-followup", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "meeting-followup", "repetition": 3 }, "trialId": "meeting-followup-mlx-qwen3-8-27b-q4-2026-11-01T02-43-15-912Z-yuuj", "success": true, "includedInCapability": true, "durationMs": 148287, "timing": { "startedAt": "2026-11-01T02:43:15.912Z", "finishedAt": "2026-11-01T02:45:44.199Z", "timeToFirstArtifactMs": 50404, "timeToLastArtifactWriteMs": 142435, "timeToFirstTokenMs": 21818, "firstTurnTtftMs": 20183, "timeToFirstToolCallMs": 31449 }, "metrics": { "process": { "sampleCount": 30, "peakRssMb": 17950, "peakCpuPercent": 119.7, "firstAtMs": 0, "lastAtMs": 145779 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 81970, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12718, "totalOutputTokens": 546, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 2883, "todayTokensCached": 0, "todayTokensOut": 512, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 2883, "totalTokensCached": 0, "totalTokensOut": 512, "totalCost": 0, "medianOutputTokensPerSec": 37.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.9, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 2883, "cachedInputTokens": 0, "outputTokens": 512, "cost": 0, "durationMs": 48771, "outputTokensPerSec": 37.91983474185421, "at": "2026-11-01T02:44:06.318Z" }, "lastUpdated": "2026-11-01T02:44:06.318Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:45:43.281Z" } }, "source": "engine-log" }, "derived": { "durationMs": 146302, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 546, "promptTokensPerSec": null, "peakRssMb": 17950, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 2883, "output": 512 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 704, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-meeting-followup", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "meeting-followup", "repetition": 3 }, "trialId": "meeting-followup-qwen3-8-27b-q4-2026-11-01T02-46-06-353Z-ccbt", "success": true, "includedInCapability": true, "durationMs": 139842, "timing": { "startedAt": "2026-11-01T02:46:06.353Z", "finishedAt": "2026-11-01T02:48:26.195Z", "timeToFirstArtifactMs": 37643, "timeToLastArtifactWriteMs": 136322, "timeToFirstTokenMs": 20043, "firstTurnTtftMs": 18245, "timeToFirstToolCallMs": 21702 }, "metrics": { "process": { "sampleCount": 28, "peakRssMb": 38566, "peakCpuPercent": 96.6, "firstAtMs": 0, "lastAtMs": 135736 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 85995, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12252, "totalOutputTokens": 963, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 2822, "todayTokensCached": 0, "todayTokensOut": 492, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 2822, "totalTokensCached": 0, "totalTokensOut": 492, "totalCost": 0, "medianOutputTokensPerSec": 45.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 45.5, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 2822, "outputTokens": 492, "cost": 0, "durationMs": 35847, "outputTokensPerSec": 45.47659752062702, "at": "2026-11-01T02:46:43.998Z" }, "lastUpdated": "2026-11-01T02:46:43.998Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:48:23.665Z" } }, "source": "engine-log" }, "derived": { "durationMs": 136260, "meanTokensPerSec": 52.33, "genTokensPerSec": 52.33, "genTokens": 963, "promptTokensPerSec": 609.4, "peakRssMb": 38566, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 2822, "output": 492 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-meeting-followup", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "meeting-followup", "repetition": 3 }, "trialId": "meeting-followup-mlx-gemma4-12b-q4-2026-11-01T02-48-48-435Z-8b3g", "success": true, "includedInCapability": true, "durationMs": 525725, "timing": { "startedAt": "2026-11-01T02:48:48.436Z", "finishedAt": "2026-11-01T02:57:34.161Z", "timeToFirstArtifactMs": 27269, "timeToLastArtifactWriteMs": 519514, "timeToFirstTokenMs": 11569, "firstTurnTtftMs": 9938, "timeToFirstToolCallMs": 20601 }, "metrics": { "process": { "sampleCount": 105, "peakRssMb": 10888, "peakCpuPercent": 104.1, "firstAtMs": 0, "lastAtMs": 522353 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 105, "peakUsedMb": 65453, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 74484, "totalOutputTokens": 10773, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 51220, "todayTokensCached": 17109, "todayTokensOut": 1185, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 51220, "totalTokensCached": 17109, "totalTokensOut": 1185, "totalCost": 0, "medianOutputTokensPerSec": 39, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5938, "cachedInputTokens": 1143, "outputTokens": 50, "cost": 0, "durationMs": 11104, "outputTokensPerSec": 44.10939643695552, "at": "2026-11-01T02:57:27.952Z" }, "lastUpdated": "2026-11-01T02:57:27.952Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T02:57:32.394Z" } }, "source": "engine-log" }, "derived": { "durationMs": 522898, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 10773, "promptTokensPerSec": null, "peakRssMb": 10888, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 51220, "output": 1185 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 927, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-meeting-followup", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "meeting-followup", "repetition": 3 }, "trialId": "meeting-followup-gemma4-12b-q4-2026-11-01T02-58-00-194Z-7248", "success": true, "includedInCapability": true, "durationMs": 177285, "timing": { "startedAt": "2026-11-01T02:58:00.194Z", "finishedAt": "2026-11-01T03:00:57.479Z", "timeToFirstArtifactMs": 25537, "timeToLastArtifactWriteMs": 170915, "timeToFirstTokenMs": 12462, "firstTurnTtftMs": 10715, "timeToFirstToolCallMs": 17542 }, "metrics": { "process": { "sampleCount": 35, "peakRssMb": 19936, "peakCpuPercent": 117.7, "firstAtMs": 0, "lastAtMs": 170859 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 35, "peakUsedMb": 64673, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 23282, "totalOutputTokens": 5266, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 32709, "todayTokensCached": 27030, "todayTokensOut": 2315, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 32709, "totalTokensCached": 27030, "totalTokensOut": 2315, "totalCost": 0, "medianOutputTokensPerSec": 102.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 102.7, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10056, "cachedInputTokens": 9442, "outputTokens": 1946, "cost": 0, "durationMs": 19136, "outputTokensPerSec": 105.46373161914914, "at": "2026-11-01T03:00:51.109Z" }, "lastUpdated": "2026-11-01T03:00:51.109Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:00:52.543Z" } }, "source": "engine-log" }, "derived": { "durationMs": 171401, "meanTokensPerSec": 101.2, "genTokensPerSec": 101.2, "genTokens": 5266, "promptTokensPerSec": 1143.19, "peakRssMb": 19936, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32709, "output": 2315 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-productivity-craftbook-spreadsheet-model", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 3 }, "trialId": "craftbook-spreadsheet-model-qwen3-8-27b-q4-2026-11-01T03-01-19-852Z-02cg", "success": true, "includedInCapability": true, "durationMs": 816207, "timing": { "startedAt": "2026-11-01T03:01:19.852Z", "finishedAt": "2026-11-01T03:14:56.059Z", "timeToFirstArtifactMs": 65192, "timeToLastArtifactWriteMs": 805868, "timeToFirstTokenMs": 25931, "firstTurnTtftMs": 19244, "timeToFirstToolCallMs": 28256 }, "metrics": { "process": { "sampleCount": 162, "peakRssMb": 42745, "peakCpuPercent": 117.9, "firstAtMs": 0, "lastAtMs": 807627 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 162, "peakUsedMb": 94274, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44446, "totalOutputTokens": 18042, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 44406, "todayTokensCached": 41512, "todayTokensOut": 3658, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 44406, "totalTokensCached": 41512, "totalTokensOut": 3658, "totalCost": 0, "medianOutputTokensPerSec": 27.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27.1, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 19218, "cachedInputTokens": 16909, "outputTokens": 2199, "cost": 0, "durationMs": 414160, "outputTokensPerSec": 27.144481665810652, "at": "2026-11-01T03:14:45.720Z" }, "lastUpdated": "2026-11-01T03:14:45.720Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:14:49.059Z" } }, "source": "engine-log" }, "derived": { "durationMs": 808151, "meanTokensPerSec": 26.83, "genTokensPerSec": 26.83, "genTokens": 18042, "promptTokensPerSec": 404.99, "peakRssMb": 42745, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44406, "output": 3658 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-craftbook-spreadsheet-model", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 3 }, "trialId": "craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-11-01T03-15-16-907Z-3wr5", "success": true, "includedInCapability": true, "durationMs": 670225, "timing": { "startedAt": "2026-11-01T03:15:16.908Z", "finishedAt": "2026-11-01T03:26:27.133Z", "timeToFirstArtifactMs": 46884, "timeToLastArtifactWriteMs": 629103, "timeToFirstTokenMs": 20338, "firstTurnTtftMs": 13831, "timeToFirstToolCallMs": 46883 }, "metrics": { "process": { "sampleCount": 133, "peakRssMb": 12825, "peakCpuPercent": 99.6, "firstAtMs": 0, "lastAtMs": 662472 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 133, "peakUsedMb": 66062, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 71842, "totalOutputTokens": 18608, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 67989, "todayTokensCached": 56613, "todayTokensOut": 3616, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 67989, "totalTokensCached": 56613, "totalTokensOut": 3616, "totalCost": 0, "medianOutputTokensPerSec": 37.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.8, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8185, "cachedInputTokens": 7981, "outputTokens": 39, "cost": 0, "durationMs": 20003, "outputTokensPerSec": 42.196269603878534, "at": "2026-11-01T03:26:08.586Z" }, "lastUpdated": "2026-11-01T03:26:08.586Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:26:20.974Z" } }, "source": "engine-log" }, "derived": { "durationMs": 662999, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 18608, "promptTokensPerSec": null, "peakRssMb": 12825, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 67989, "output": 3616 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1262, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-craftbook-spreadsheet-model", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 3 }, "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-11-01T03-26-49-157Z-c17x", "success": true, "includedInCapability": true, "durationMs": 366990, "timing": { "startedAt": "2026-11-01T03:26:49.157Z", "finishedAt": "2026-11-01T03:32:56.147Z", "timeToFirstArtifactMs": 30174, "timeToLastArtifactWriteMs": 311282, "timeToFirstTokenMs": 18369, "firstTurnTtftMs": 11686, "timeToFirstToolCallMs": 30173 }, "metrics": { "process": { "sampleCount": 73, "peakRssMb": 24829, "peakCpuPercent": 104.4, "firstAtMs": 0, "lastAtMs": 361190 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 73, "peakUsedMb": 79449, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 62449, "totalOutputTokens": 16860, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43813, "todayTokensCached": 32719, "todayTokensOut": 1161, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43813, "totalTokensCached": 32719, "totalTokensOut": 1161, "totalCost": 0, "medianOutputTokensPerSec": 66, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 66, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8287, "cachedInputTokens": 8054, "outputTokens": 35, "cost": 0, "durationMs": 30887, "outputTokensPerSec": 83.07494581803167, "at": "2026-11-01T03:32:36.677Z" }, "lastUpdated": "2026-11-01T03:32:36.677Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:32:51.923Z" } }, "source": "engine-log" }, "derived": { "durationMs": 361713, "meanTokensPerSec": 67.14, "genTokensPerSec": 67.14, "genTokens": 16860, "promptTokensPerSec": 979.02, "peakRssMb": 24829, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43813, "output": 1161 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-craftbook-spreadsheet-model", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 3 }, "trialId": "craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-11-01T03-33-17-724Z-hzpq", "success": true, "includedInCapability": true, "durationMs": 949852, "timing": { "startedAt": "2026-11-01T03:33:17.725Z", "finishedAt": "2026-11-01T03:49:07.577Z", "timeToFirstArtifactMs": 106387, "timeToLastArtifactWriteMs": 942587, "timeToFirstTokenMs": 27286, "firstTurnTtftMs": 20780, "timeToFirstToolCallMs": 35546 }, "metrics": { "process": { "sampleCount": 189, "peakRssMb": 18581, "peakCpuPercent": 111.5, "firstAtMs": 0, "lastAtMs": 943147 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 189, "peakUsedMb": 82640, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 38625, "totalOutputTokens": 21577, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 38449, "todayTokensCached": 35974, "todayTokensOut": 4388, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 38449, "totalTokensCached": 35974, "totalTokensOut": 4388, "totalCost": 0, "medianOutputTokensPerSec": 26.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.9, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 19068, "cachedInputTokens": 17801, "outputTokens": 2092, "cost": 0, "durationMs": 490857, "outputTokensPerSec": 24.02019517122486, "at": "2026-11-01T03:49:00.312Z" }, "lastUpdated": "2026-11-01T03:49:00.312Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:49:02.487Z" } }, "source": "engine-log" }, "derived": { "durationMs": 943694, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 21577, "promptTokensPerSec": null, "peakRssMb": 18581, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 38449, "output": 4388 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 562, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-productivity-conflict-synthesis", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T03-49-29-133Z-8slx", "success": true, "includedInCapability": true, "durationMs": 47710, "timing": { "startedAt": "2026-11-01T03:49:29.133Z", "finishedAt": "2026-11-01T03:50:16.843Z", "timeToFirstArtifactMs": 39960, "timeToLastArtifactWriteMs": 41340, "timeToFirstTokenMs": 17163, "firstTurnTtftMs": 15541, "timeToFirstToolCallMs": 27640 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 10291, "peakCpuPercent": 114.5, "firstAtMs": 0, "lastAtMs": 45153 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 65731, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 11132, "totalOutputTokens": 746, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11100, "todayTokensCached": 10640, "todayTokensOut": 26, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11100, "totalTokensCached": 10640, "totalTokensOut": 26, "totalCost": 0, "medianOutputTokensPerSec": 40.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11100, "cachedInputTokens": 10640, "outputTokens": 26, "cost": 0, "durationMs": 39718, "outputTokensPerSec": 40.430558486733226, "at": "2026-11-01T03:50:10.473Z" }, "lastUpdated": "2026-11-01T03:50:10.473Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:50:15.885Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45698, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 746, "promptTokensPerSec": null, "peakRssMb": 10291, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 11100, "output": 26 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1081, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-conflict-synthesis", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-gemma4-12b-q4-2026-11-01T03-50-38-759Z-sq9b", "success": true, "includedInCapability": true, "durationMs": 116010, "timing": { "startedAt": "2026-11-01T03:50:38.760Z", "finishedAt": "2026-11-01T03:52:34.770Z", "timeToFirstArtifactMs": 24405, "timeToLastArtifactWriteMs": 106850, "timeToFirstTokenMs": 14500, "firstTurnTtftMs": 12646, "timeToFirstToolCallMs": 19442 }, "metrics": { "process": { "sampleCount": 23, "peakRssMb": 21959, "peakCpuPercent": 115.9, "firstAtMs": 1, "lastAtMs": 110386 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 66989, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 86685, "totalOutputTokens": 1891, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 25808, "todayTokensCached": 17203, "todayTokensOut": 920, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 25808, "totalTokensCached": 17203, "totalTokensOut": 920, "totalCost": 0, "medianOutputTokensPerSec": 92.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 92.5, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8213, "cachedInputTokens": 6536, "outputTokens": 387, "cost": 0, "durationMs": 40412, "outputTokensPerSec": 92.50059789894239, "at": "2026-11-01T03:52:25.610Z" }, "lastUpdated": "2026-11-01T03:52:25.610Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:52:30.737Z" } }, "source": "engine-log" }, "derived": { "durationMs": 110929, "meanTokensPerSec": 89.31, "genTokensPerSec": 89.31, "genTokens": 1891, "promptTokensPerSec": 1162.37, "peakRssMb": 21959, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 25808, "output": 920 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-conflict-synthesis", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T03-52-56-729Z-jhrw", "success": true, "includedInCapability": true, "durationMs": 413576, "timing": { "startedAt": "2026-11-01T03:52:56.730Z", "finishedAt": "2026-11-01T03:59:50.306Z", "timeToFirstArtifactMs": 188250, "timeToLastArtifactWriteMs": 405059, "timeToFirstTokenMs": 27689, "firstTurnTtftMs": 26042, "timeToFirstToolCallMs": 32480 }, "metrics": { "process": { "sampleCount": 82, "peakRssMb": 17986, "peakCpuPercent": 120.4, "firstAtMs": 0, "lastAtMs": 406330 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 82, "peakUsedMb": 81939, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 14143, "totalOutputTokens": 9030, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 14031, "todayTokensCached": 12986, "todayTokensOut": 731, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 14031, "totalTokensCached": 12986, "totalTokensOut": 731, "totalCost": 0, "medianOutputTokensPerSec": 24.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.5, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14031, "cachedInputTokens": 12986, "outputTokens": 731, "cost": 0, "durationMs": 403411, "outputTokensPerSec": 24.494840917837813, "at": "2026-11-01T03:59:41.788Z" }, "lastUpdated": "2026-11-01T03:59:41.788Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T03:59:44.653Z" } }, "source": "engine-log" }, "derived": { "durationMs": 406854, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 9030, "promptTokensPerSec": null, "peakRssMb": 17986, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 14031, "output": 731 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 583, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-conflict-synthesis", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 3 }, "trialId": "conflict-synthesis-qwen3-8-27b-q4-2026-11-01T04-00-13-052Z-n4et", "success": false, "includedInCapability": true, "durationMs": 600717, "timing": { "startedAt": "2026-11-01T04:00:13.052Z", "finishedAt": "2026-11-01T04:10:13.769Z", "timeToFirstArtifactMs": 96222, "timeToLastArtifactWriteMs": 596877, "timeToFirstTokenMs": 35129, "firstTurnTtftMs": 33309, "timeToFirstToolCallMs": 39673 }, "metrics": { "process": { "sampleCount": 120, "peakRssMb": 48372, "peakCpuPercent": 106.8, "firstAtMs": 0, "lastAtMs": 596986 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 120, "peakUsedMb": 94807, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 158627, "totalOutputTokens": 6328, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 49877, "todayTokensCached": 20561, "todayTokensOut": 3744, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 49877, "totalTokensCached": 20561, "totalTokensOut": 3744, "totalCost": 0, "medianOutputTokensPerSec": 37.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.3, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11462, "outputTokens": 880, "cost": 0, "durationMs": 194429, "outputTokensPerSec": 37.573990625396206, "at": "2026-11-01T04:10:09.929Z" }, "lastUpdated": "2026-11-01T04:10:09.929Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T04:10:11.633Z" } }, "source": "engine-log" }, "derived": { "durationMs": 597530, "meanTokensPerSec": 34.94, "genTokensPerSec": 34.94, "genTokens": 6328, "promptTokensPerSec": 399.52, "peakRssMb": 48372, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 49877, "output": 3744 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-productivity-docblocks-theme-roundtrip", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 3 }, "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-11-01T04-10-50-055Z-s6hw", "success": false, "includedInCapability": true, "durationMs": 502863, "timing": { "startedAt": "2026-11-01T04:10:50.056Z", "finishedAt": "2026-11-01T04:19:12.919Z", "timeToFirstArtifactMs": 99221, "timeToLastArtifactWriteMs": 494464, "timeToFirstTokenMs": 9247, "firstTurnTtftMs": 7499, "timeToFirstToolCallMs": 99223 }, "metrics": { "process": { "sampleCount": 100, "peakRssMb": 26341, "peakCpuPercent": 117.2, "firstAtMs": 0, "lastAtMs": 497684 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 100, "peakUsedMb": 74082, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 57110, "totalOutputTokens": 10617, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 86140, "todayTokensCached": 50727, "todayTokensOut": 3805, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 86140, "totalTokensCached": 50727, "totalTokensOut": 3805, "totalCost": 0, "medianOutputTokensPerSec": 100.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100.1, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11779, "cachedInputTokens": 11051, "outputTokens": 1548, "cost": 0, "durationMs": 20778, "outputTokensPerSec": 77.95160711825656, "at": "2026-11-01T04:19:04.520Z" }, "lastUpdated": "2026-11-01T04:19:04.520Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T04:19:09.327Z" } }, "source": "engine-log" }, "derived": { "durationMs": 498230, "meanTokensPerSec": 89.53, "genTokensPerSec": 89.53, "genTokens": 10617, "promptTokensPerSec": 1208.73, "peakRssMb": 26341, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 86140, "output": 3805 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-docblocks-theme-roundtrip", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 3 }, "trialId": "docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-11-01T04-19-58-506Z-xpc0", "success": false, "includedInCapability": true, "durationMs": 319066, "timing": { "startedAt": "2026-11-01T04:19:58.507Z", "finishedAt": "2026-11-01T04:25:17.573Z", "timeToFirstArtifactMs": 80023, "timeToLastArtifactWriteMs": 314655, "timeToFirstTokenMs": 16677, "firstTurnTtftMs": 15054, "timeToFirstToolCallMs": 21219 }, "metrics": { "process": { "sampleCount": 64, "peakRssMb": 18003, "peakCpuPercent": 119.8, "firstAtMs": 0, "lastAtMs": 316131 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 64, "peakUsedMb": 82126, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 84978, "totalOutputTokens": 2554, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 55044, "todayTokensCached": 14518, "todayTokensOut": 2842, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 55044, "totalTokensCached": 14518, "totalTokensOut": 2842, "totalCost": 0, "medianOutputTokensPerSec": 33.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.2, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 10808, "cachedInputTokens": 1220, "outputTokens": 207, "cost": 0, "durationMs": 42559, "outputTokensPerSec": 31.907695273733157, "at": "2026-11-01T04:25:13.164Z" }, "lastUpdated": "2026-11-01T04:25:13.164Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T04:25:16.238Z" } }, "source": "engine-log" }, "derived": { "durationMs": 316674, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 2554, "promptTokensPerSec": null, "peakRssMb": 18003, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 55044, "output": 2842 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 537.5, "sampleCount": 8, "structuredSampleCount": 0, "firstTokenSampleCount": 8, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-docblocks-theme-roundtrip", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 3 }, "trialId": "docblocks-theme-roundtrip-qwen3-8-27b-q4-2026-11-01T04-25-42-241Z-8bf8", "success": false, "includedInCapability": true, "durationMs": 223888, "timing": { "startedAt": "2026-11-01T04:25:42.241Z", "finishedAt": "2026-11-01T04:29:26.129Z", "timeToFirstArtifactMs": 65138, "timeToLastArtifactWriteMs": 215949, "timeToFirstTokenMs": 20091, "firstTurnTtftMs": 18270, "timeToFirstToolCallMs": 22021 }, "metrics": { "process": { "sampleCount": 44, "peakRssMb": 41506, "peakCpuPercent": 92, "firstAtMs": 0, "lastAtMs": 215752 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 44, "peakUsedMb": 89343, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 25305, "totalOutputTokens": 5143, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 68634, "todayTokensCached": 60896, "todayTokensOut": 4906, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 68634, "totalTokensCached": 60896, "totalTokensOut": 4906, "totalCost": 0, "medianOutputTokensPerSec": 40, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 40, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13184, "cachedInputTokens": 12400, "outputTokens": 699, "cost": 0, "durationMs": 24145, "outputTokensPerSec": 39.86745499132826, "at": "2026-11-01T04:29:18.190Z" }, "lastUpdated": "2026-11-01T04:29:18.190Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T04:29:19.588Z" } }, "source": "engine-log" }, "derived": { "durationMs": 216294, "meanTokensPerSec": 37.82, "genTokensPerSec": 37.82, "genTokens": 5143, "promptTokensPerSec": 417.24, "peakRssMb": 41506, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 68634, "output": 4906 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-docblocks-theme-roundtrip", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 3 }, "trialId": "docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-11-01T04-30-05-917Z-74to", "success": false, "includedInCapability": true, "durationMs": 324049, "timing": { "startedAt": "2026-11-01T04:30:05.918Z", "finishedAt": "2026-11-01T04:35:29.967Z", "timeToFirstArtifactMs": 109115, "timeToLastArtifactWriteMs": 321514, "timeToFirstTokenMs": 10121, "firstTurnTtftMs": 8394, "timeToFirstToolCallMs": 106999 }, "metrics": { "process": { "sampleCount": 65, "peakRssMb": 10557, "peakCpuPercent": 94.3, "firstAtMs": 0, "lastAtMs": 321332 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 65, "peakUsedMb": 64797, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 72209, "totalOutputTokens": 6024, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 70891, "todayTokensCached": 25917, "todayTokensOut": 1894, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 70891, "totalTokensCached": 25917, "totalTokensOut": 1894, "totalCost": 0, "medianOutputTokensPerSec": 40.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.8, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11621, "cachedInputTokens": 523, "outputTokens": 133, "cost": 0, "durationMs": 14025, "outputTokensPerSec": 34.17508391910253, "at": "2026-11-01T04:35:27.432Z" }, "lastUpdated": "2026-11-01T04:35:27.432Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T04:35:28.947Z" } }, "source": "engine-log" }, "derived": { "durationMs": 321872, "meanTokensPerSec": 46, "genTokensPerSec": 46, "genTokens": 6024, "promptTokensPerSec": null, "peakRssMb": 10557, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 70891, "output": 1894 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1150.5, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-productivity-craftbook-powerpoint-deck", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 3 }, "trialId": "craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-11-01T04-35-51-551Z-qzuu", "success": true, "includedInCapability": true, "durationMs": 1059329, "timing": { "startedAt": "2026-11-01T04:35:51.552Z", "finishedAt": "2026-11-01T04:53:30.881Z", "timeToFirstArtifactMs": 305214, "timeToLastArtifactWriteMs": 1053377, "timeToFirstTokenMs": 33625, "firstTurnTtftMs": 27095, "timeToFirstToolCallMs": 40813 }, "metrics": { "process": { "sampleCount": 211, "peakRssMb": 19721, "peakCpuPercent": 113.5, "firstAtMs": 0, "lastAtMs": 1053514 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 211, "peakUsedMb": 87197, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 108808, "totalOutputTokens": 14893, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 46405, "todayTokensCached": 44375, "todayTokensOut": 5470, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 46405, "totalTokensCached": 44375, "totalTokensOut": 5470, "totalCost": 0, "medianOutputTokensPerSec": 27, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12561, "cachedInputTokens": 12486, "outputTokens": 74, "cost": 0, "durationMs": 240048, "outputTokensPerSec": 26.946084439472415, "at": "2026-11-01T04:47:58.266Z" }, "lastUpdated": "2026-11-01T04:47:58.266Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T04:53:28.174Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1055550, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 14893, "promptTokensPerSec": null, "peakRssMb": 19721, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 46405, "output": 5470 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 519, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-craftbook-powerpoint-deck", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 3 }, "trialId": "craftbook-powerpoint-deck-qwen3-8-27b-q4-2026-11-01T04-53-53-422Z-upzs", "success": true, "includedInCapability": true, "durationMs": 1383799, "timing": { "startedAt": "2026-11-01T04:53:53.423Z", "finishedAt": "2026-11-01T05:16:57.222Z", "timeToFirstArtifactMs": 117374, "timeToLastArtifactWriteMs": 1370328, "timeToFirstTokenMs": 37039, "firstTurnTtftMs": 30315, "timeToFirstToolCallMs": 39077 }, "metrics": { "process": { "sampleCount": 274, "peakRssMb": 51335, "peakCpuPercent": 141.3, "firstAtMs": 0, "lastAtMs": 1369827 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 274, "peakUsedMb": 100014, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 143933, "totalOutputTokens": 23353, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 101930, "todayTokensCached": 95152, "todayTokensOut": 3355, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 101930, "totalTokensCached": 95152, "totalTokensOut": 3355, "totalCost": 0, "medianOutputTokensPerSec": 31.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.1, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14257, "cachedInputTokens": 12941, "outputTokens": 109, "cost": 0, "durationMs": 150289, "outputTokensPerSec": 27.409177912466255, "at": "2026-11-01T05:13:20.561Z" }, "lastUpdated": "2026-11-01T05:13:20.561Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T05:16:47.093Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1372619, "meanTokensPerSec": 28.42, "genTokensPerSec": 28.42, "genTokens": 23353, "promptTokensPerSec": 396.84, "peakRssMb": 51335, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 101930, "output": 3355 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-craftbook-powerpoint-deck", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 3 }, "trialId": "craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-11-01T05-17-19-131Z-jyul", "success": false, "includedInCapability": true, "durationMs": 1109420, "timing": { "startedAt": "2026-11-01T05:17:19.131Z", "finishedAt": "2026-11-01T05:35:48.551Z", "timeToFirstArtifactMs": 41134, "timeToLastArtifactWriteMs": 736135, "timeToFirstTokenMs": 21921, "firstTurnTtftMs": 15404, "timeToFirstToolCallMs": 32141 }, "metrics": { "process": { "sampleCount": 221, "peakRssMb": 12359, "peakCpuPercent": 135.6, "firstAtMs": 0, "lastAtMs": 1104131 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 221, "peakUsedMb": 91341, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 189162, "totalOutputTokens": 22230, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 88958, "todayTokensCached": 87636, "todayTokensOut": 1641, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 88958, "totalTokensCached": 87636, "totalTokensOut": 1641, "totalCost": 0, "medianOutputTokensPerSec": 37.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.5, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 25547, "cachedInputTokens": 25459, "outputTokens": 285, "cost": 0, "durationMs": 65510, "outputTokensPerSec": 34.676078846401076, "at": "2026-11-01T05:28:31.722Z" }, "lastUpdated": "2026-11-01T05:28:31.722Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T05:35:46.878Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1106680, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 22230, "promptTokensPerSec": null, "peakRssMb": 12359, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 88958, "output": 1641 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1109, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-craftbook-powerpoint-deck", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 3 }, "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-11-01T05-36-11-271Z-4whh", "success": true, "includedInCapability": true, "durationMs": 258023, "timing": { "startedAt": "2026-11-01T05:36:11.271Z", "finishedAt": "2026-11-01T05:40:29.294Z", "timeToFirstArtifactMs": 30920, "timeToLastArtifactWriteMs": 244749, "timeToFirstTokenMs": 20291, "firstTurnTtftMs": 13553, "timeToFirstToolCallMs": 25610 }, "metrics": { "process": { "sampleCount": 50, "peakRssMb": 29169, "peakCpuPercent": 148, "firstAtMs": 0, "lastAtMs": 245786 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 50, "peakUsedMb": 92233, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 85534, "totalOutputTokens": 7785, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 64357, "todayTokensCached": 62224, "todayTokensOut": 972, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 64357, "totalTokensCached": 62224, "totalTokensOut": 972, "totalCost": 0, "medianOutputTokensPerSec": 86.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 86.6, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11749, "cachedInputTokens": 10919, "outputTokens": 39, "cost": 0, "durationMs": 35269, "outputTokensPerSec": 95.97583429519642, "at": "2026-11-01T05:39:28.069Z" }, "lastUpdated": "2026-11-01T05:39:28.069Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T05:40:19.130Z" } }, "source": "engine-log" }, "derived": { "durationMs": 246808, "meanTokensPerSec": 77.53, "genTokensPerSec": 77.53, "genTokens": 7785, "promptTokensPerSec": 1029.63, "peakRssMb": 29169, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 64357, "output": 972 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-productivity-wikipedia-research-brief", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 3 }, "trialId": "wikipedia-research-brief-qwen3-8-27b-q4-2026-11-01T05-40-52-417Z-rpv3", "success": true, "includedInCapability": true, "durationMs": 400879, "timing": { "startedAt": "2026-11-01T05:40:52.418Z", "finishedAt": "2026-11-01T05:47:33.297Z", "timeToFirstArtifactMs": 331045, "timeToLastArtifactWriteMs": 395288, "timeToFirstTokenMs": 14942, "firstTurnTtftMs": 13064, "timeToFirstToolCallMs": 77570 }, "metrics": { "process": { "sampleCount": 80, "peakRssMb": 40172, "peakCpuPercent": 113.7, "firstAtMs": 0, "lastAtMs": 396332 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 80, "peakUsedMb": 86814, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 27774, "totalOutputTokens": 8554, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 34162, "todayTokensCached": 29943, "todayTokensOut": 450, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 34162, "totalTokensCached": 29943, "totalTokensOut": 450, "totalCost": 0, "medianOutputTokensPerSec": 35, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 35, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14098, "cachedInputTokens": 12032, "outputTokens": 137, "cost": 0, "durationMs": 17901, "outputTokensPerSec": 34.98619074471194, "at": "2026-11-01T05:47:27.709Z" }, "lastUpdated": "2026-11-01T05:47:27.709Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T05:47:30.367Z" } }, "source": "engine-log" }, "derived": { "durationMs": 396875, "meanTokensPerSec": 27.51, "genTokensPerSec": 27.51, "genTokens": 8554, "promptTokensPerSec": 393.58, "peakRssMb": 40172, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 34162, "output": 450 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-wikipedia-research-brief", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 3 }, "trialId": "wikipedia-research-brief-mlx-gemma4-12b-q4-2026-11-01T05-47-58-329Z-iy7j", "success": false, "includedInCapability": true, "durationMs": 740887, "timing": { "startedAt": "2026-11-01T05:47:58.329Z", "finishedAt": "2026-11-01T06:00:19.216Z", "timeToFirstArtifactMs": 57210, "timeToLastArtifactWriteMs": 737817, "timeToFirstTokenMs": 12052, "firstTurnTtftMs": 10359, "timeToFirstToolCallMs": 33118 }, "metrics": { "process": { "sampleCount": 148, "peakRssMb": 10514, "peakCpuPercent": 149.9, "firstAtMs": 0, "lastAtMs": 738039 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 148, "peakUsedMb": 65206, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 179695, "totalOutputTokens": 16770, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 172833, "todayTokensCached": 116972, "todayTokensOut": 10958, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 172833, "totalTokensCached": 116972, "totalTokensOut": 10958, "totalCost": 0, "medianOutputTokensPerSec": 34.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.8, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 29502, "cachedInputTokens": 29446, "outputTokens": 1442, "cost": 0, "durationMs": 73894, "outputTokensPerSec": 33.20057936758188, "at": "2026-11-01T06:00:16.149Z" }, "lastUpdated": "2026-11-01T06:00:16.149Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T06:00:18.012Z" } }, "source": "engine-log" }, "derived": { "durationMs": 738582, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 16770, "promptTokensPerSec": null, "peakRssMb": 10514, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 172833, "output": 10958 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1122, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-wikipedia-research-brief", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 3 }, "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-11-01T06-00-40-991Z-4x0l", "success": true, "includedInCapability": true, "durationMs": 167291, "timing": { "startedAt": "2026-11-01T06:00:40.991Z", "finishedAt": "2026-11-01T06:03:28.282Z", "timeToFirstArtifactMs": 51352, "timeToLastArtifactWriteMs": 155548, "timeToFirstTokenMs": 9956, "firstTurnTtftMs": 8063, "timeToFirstToolCallMs": 26802 }, "metrics": { "process": { "sampleCount": 32, "peakRssMb": 24341, "peakCpuPercent": 117.1, "firstAtMs": 0, "lastAtMs": 155574 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 69294, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 31688, "totalOutputTokens": 6748, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 79238, "todayTokensCached": 71510, "todayTokensOut": 3699, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 79238, "totalTokensCached": 71510, "totalTokensOut": 3699, "totalCost": 0, "medianOutputTokensPerSec": 65.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 65.6, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18332, "cachedInputTokens": 16897, "outputTokens": 82, "cost": 0, "durationMs": 8785, "outputTokensPerSec": 85.7807317414124, "at": "2026-11-01T06:03:16.542Z" }, "lastUpdated": "2026-11-01T06:03:16.542Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T06:03:18.174Z" } }, "source": "engine-log" }, "derived": { "durationMs": 156115, "meanTokensPerSec": 63.62, "genTokensPerSec": 63.62, "genTokens": 6748, "promptTokensPerSec": 980.76, "peakRssMb": 24341, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 79238, "output": 3699 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-wikipedia-research-brief", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 3 }, "trialId": "wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-11-01T06-03-50-420Z-dvkk", "success": false, "includedInCapability": true, "durationMs": 630218, "timing": { "startedAt": "2026-11-01T06:03:50.421Z", "finishedAt": "2026-11-01T06:14:20.639Z", "timeToFirstArtifactMs": 102353, "timeToLastArtifactWriteMs": 626046, "timeToFirstTokenMs": 13263, "firstTurnTtftMs": 11531, "timeToFirstToolCallMs": 60159 }, "metrics": { "process": { "sampleCount": 126, "peakRssMb": 18012, "peakCpuPercent": 121.2, "firstAtMs": 1, "lastAtMs": 627143 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 126, "peakUsedMb": 82007, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 139096, "totalOutputTokens": 10060, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 69826, "todayTokensCached": 10443, "todayTokensOut": 8381, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 69826, "totalTokensCached": 10443, "totalTokensOut": 8381, "totalCost": 0, "medianOutputTokensPerSec": 31.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.6, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16204, "cachedInputTokens": 1301, "outputTokens": 1559, "cost": 0, "durationMs": 148222, "outputTokensPerSec": 31.382904295664872, "at": "2026-11-01T06:14:16.467Z" }, "lastUpdated": "2026-11-01T06:14:16.467Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T06:14:19.226Z" } }, "source": "engine-log" }, "derived": { "durationMs": 627684, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 10060, "promptTokensPerSec": null, "peakRssMb": 18012, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 69826, "output": 8381 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 520, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-productivity-craftbook-week-plan", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 3 }, "trialId": "craftbook-week-plan-mlx-gemma4-12b-q4-2026-11-01T06-14-42-490Z-tv6q", "success": false, "includedInCapability": true, "durationMs": 2106203, "timing": { "startedAt": "2026-11-01T06:14:42.491Z", "finishedAt": "2026-11-01T06:49:48.694Z", "timeToFirstArtifactMs": 50537, "timeToLastArtifactWriteMs": 2092855, "timeToFirstTokenMs": 24091, "firstTurnTtftMs": 17542, "timeToFirstToolCallMs": 32864 }, "metrics": { "process": { "sampleCount": 417, "peakRssMb": 16012, "peakCpuPercent": 111.4, "firstAtMs": 0, "lastAtMs": 2089425 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 417, "peakUsedMb": 78630, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 362859, "totalOutputTokens": 30926, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 29, "todayTokensIn": 325660, "todayTokensCached": 257651, "todayTokensOut": 12363, "todayCost": 0, "totalTurns": 29, "totalTokensIn": 325660, "totalTokensCached": 257651, "totalTokensOut": 12363, "totalCost": 0, "medianOutputTokensPerSec": 38.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.5, "turns": 29 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11859, "cachedInputTokens": 1689, "outputTokens": 89, "cost": 0, "durationMs": 24431, "outputTokensPerSec": 32.32524979792973, "at": "2026-11-01T06:49:35.349Z" }, "lastUpdated": "2026-11-01T06:49:35.349Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T06:49:36.552Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2092979, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 30926, "promptTokensPerSec": null, "peakRssMb": 16012, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 325660, "output": 12363 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1252, "sampleCount": 31, "structuredSampleCount": 0, "firstTokenSampleCount": 31, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-craftbook-week-plan", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 3 }, "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-11-01T06-50-11-622Z-6540", "success": true, "includedInCapability": true, "durationMs": 378349, "timing": { "startedAt": "2026-11-01T06:50:11.622Z", "finishedAt": "2026-11-01T06:56:29.971Z", "timeToFirstArtifactMs": 47690, "timeToLastArtifactWriteMs": 366985, "timeToFirstTokenMs": 23751, "firstTurnTtftMs": 17031, "timeToFirstToolCallMs": 28751 }, "metrics": { "process": { "sampleCount": 75, "peakRssMb": 29773, "peakCpuPercent": 166, "firstAtMs": 0, "lastAtMs": 371473 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 75, "peakUsedMb": 84881, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 110385, "totalOutputTokens": 8779, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 110160, "todayTokensCached": 94037, "todayTokensOut": 2544, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 110160, "totalTokensCached": 94037, "totalTokensOut": 2544, "totalCost": 0, "medianOutputTokensPerSec": 82, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 82, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7085, "outputTokens": 86, "cost": 0, "durationMs": 9405, "outputTokensPerSec": 93.13119116984828, "at": "2026-11-01T06:56:18.610Z" }, "lastUpdated": "2026-11-01T06:56:18.610Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T06:56:24.693Z" } }, "source": "engine-log" }, "derived": { "durationMs": 371995, "meanTokensPerSec": 72.12, "genTokensPerSec": 72.12, "genTokens": 8779, "promptTokensPerSec": 1057.69, "peakRssMb": 29773, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 110160, "output": 2544 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-craftbook-week-plan", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 3 }, "trialId": "craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-11-01T06-56-53-321Z-3fxg", "success": true, "includedInCapability": true, "durationMs": 4252291, "timing": { "startedAt": "2026-11-01T06:56:53.321Z", "finishedAt": "2026-11-01T08:07:45.612Z", "timeToFirstArtifactMs": 853431, "timeToLastArtifactWriteMs": 4238575, "timeToFirstTokenMs": 32199, "firstTurnTtftMs": 25530, "timeToFirstToolCallMs": 38823 }, "metrics": { "process": { "sampleCount": 845, "peakRssMb": 20400, "peakCpuPercent": 137.6, "firstAtMs": 0, "lastAtMs": 4234957 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 845, "peakUsedMb": 85642, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 252663, "totalOutputTokens": 83723, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 245381, "todayTokensCached": 211160, "todayTokensOut": 24261, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 245381, "totalTokensCached": 211160, "totalTokensOut": 24261, "totalCost": 0, "medianOutputTokensPerSec": 27.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27.5, "turns": 16 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5366, "cachedInputTokens": 0, "outputTokens": 453, "cost": 0, "durationMs": 26587, "outputTokensPerSec": 49.90788251075839, "at": "2026-11-01T08:07:31.901Z" }, "lastUpdated": "2026-11-01T08:07:31.901Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T08:07:33.450Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4239015, "meanTokensPerSec": 26, "genTokensPerSec": 26, "genTokens": 83723, "promptTokensPerSec": null, "peakRssMb": 20400, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 245381, "output": 24261 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 555, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-craftbook-week-plan", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 3 }, "trialId": "craftbook-week-plan-qwen3-8-27b-q4-2026-11-01T08-08-08-946Z-m6ac", "success": false, "includedInCapability": true, "durationMs": 5406523, "timing": { "startedAt": "2026-11-01T08:08:08.946Z", "finishedAt": "2026-11-01T09:38:15.469Z", "timeToFirstArtifactMs": 500989, "timeToLastArtifactWriteMs": 4601721, "timeToFirstTokenMs": 44102, "firstTurnTtftMs": 37296, "timeToFirstToolCallMs": 55151 }, "metrics": { "process": { "sampleCount": 1078, "peakRssMb": 49554, "peakCpuPercent": 127.5, "firstAtMs": 0, "lastAtMs": 5403298 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1078, "peakUsedMb": 101809, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 459556, "totalOutputTokens": 86121, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 19, "todayTokensIn": 392072, "todayTokensCached": 340479, "todayTokensOut": 25392, "todayCost": 0, "totalTurns": 19, "totalTokensIn": 392072, "totalTokensCached": 340479, "totalTokensOut": 25392, "totalCost": 0, "medianOutputTokensPerSec": 23.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.2, "turns": 19 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 29072, "cachedInputTokens": 28813, "outputTokens": 184, "cost": 0, "durationMs": 80550, "outputTokensPerSec": 22.555737631166853, "at": "2026-11-01T09:32:42.577Z" }, "lastUpdated": "2026-11-01T09:32:42.577Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T09:38:13.843Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5403823, "meanTokensPerSec": 22.78, "genTokensPerSec": 22.78, "genTokens": 86121, "promptTokensPerSec": 383.49, "peakRssMb": 49554, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 392072, "output": 25392 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-productivity-craftbook-ab-test-readout", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 3 }, "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-11-01T09-38-39-931Z-4qof", "success": false, "includedInCapability": true, "durationMs": 4313931, "timing": { "startedAt": "2026-11-01T09:38:39.931Z", "finishedAt": "2026-11-01T10:50:33.862Z", "timeToFirstArtifactMs": 40381, "timeToLastArtifactWriteMs": 4008743, "timeToFirstTokenMs": 21685, "firstTurnTtftMs": 14994, "timeToFirstToolCallMs": 28631 }, "metrics": { "process": { "sampleCount": 859, "peakRssMb": 33786, "peakCpuPercent": 166.9, "firstAtMs": 0, "lastAtMs": 4309504 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 859, "peakUsedMb": 92073, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 618586, "totalOutputTokens": 124725, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 71, "todayTokensIn": 786312, "todayTokensCached": 723914, "todayTokensOut": 25904, "todayCost": 0, "totalTurns": 71, "totalTokensIn": 786312, "totalTokensCached": 723914, "totalTokensOut": 25904, "totalCost": 0, "medianOutputTokensPerSec": 78.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 78.4, "turns": 71 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15804, "cachedInputTokens": 15215, "outputTokens": 84, "cost": 0, "durationMs": 29014, "outputTokensPerSec": 87.20475150323131, "at": "2026-11-01T10:45:28.674Z" }, "lastUpdated": "2026-11-01T10:45:28.674Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T10:50:33.546Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4312567, "meanTokensPerSec": 71.22, "genTokensPerSec": 71.22, "genTokens": 124725, "promptTokensPerSec": 1046.55, "peakRssMb": 33786, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 786312, "output": 25904 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-craftbook-ab-test-readout", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 3 }, "trialId": "craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-11-01T10-50-56-405Z-bnqy", "success": true, "includedInCapability": true, "durationMs": 1921154, "timing": { "startedAt": "2026-11-01T10:50:56.406Z", "finishedAt": "2026-11-01T11:22:57.560Z", "timeToFirstArtifactMs": 78713, "timeToLastArtifactWriteMs": 1910437, "timeToFirstTokenMs": 28827, "firstTurnTtftMs": 22361, "timeToFirstToolCallMs": 33204 }, "metrics": { "process": { "sampleCount": 382, "peakRssMb": 20112, "peakCpuPercent": 181.1, "firstAtMs": 0, "lastAtMs": 1912321 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 382, "peakUsedMb": 82795, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 107712, "totalOutputTokens": 42010, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 109666, "todayTokensCached": 106327, "todayTokensOut": 13920, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 109666, "totalTokensCached": 106327, "totalTokensOut": 13920, "totalCost": 0, "medianOutputTokensPerSec": 30.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.3, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8501, "cachedInputTokens": 8456, "outputTokens": 256, "cost": 0, "durationMs": 37132, "outputTokensPerSec": 36.22078678808476, "at": "2026-11-01T11:22:15.705Z" }, "lastUpdated": "2026-11-01T11:22:15.705Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T11:22:50.461Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1913096, "meanTokensPerSec": 30, "genTokensPerSec": 30, "genTokens": 42010, "promptTokensPerSec": null, "peakRssMb": 20112, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 109666, "output": 13920 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 626, "sampleCount": 10, "structuredSampleCount": 0, "firstTokenSampleCount": 10, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-craftbook-ab-test-readout", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 3 }, "trialId": "craftbook-ab-test-readout-qwen3-8-27b-q4-2026-11-01T11-23-22-326Z-a1oy", "success": true, "includedInCapability": true, "durationMs": 1494530, "timing": { "startedAt": "2026-11-01T11:23:22.327Z", "finishedAt": "2026-11-01T11:48:16.857Z", "timeToFirstArtifactMs": 60861, "timeToLastArtifactWriteMs": 1486909, "timeToFirstTokenMs": 37443, "firstTurnTtftMs": 30616, "timeToFirstToolCallMs": 38707 }, "metrics": { "process": { "sampleCount": 298, "peakRssMb": 47902, "peakCpuPercent": 111.5, "firstAtMs": 0, "lastAtMs": 1489949 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 298, "peakUsedMb": 101002, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 142753, "totalOutputTokens": 28044, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 115230, "todayTokensCached": 111478, "todayTokensOut": 8371, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 115230, "totalTokensCached": 111478, "totalTokensOut": 8371, "totalCost": 0, "medianOutputTokensPerSec": 34.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 34.3, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 13272, "cachedInputTokens": 13057, "outputTokens": 415, "cost": 0, "durationMs": 172122, "outputTokensPerSec": 29.550964528492592, "at": "2026-11-01T11:47:31.576Z" }, "lastUpdated": "2026-11-01T11:47:31.576Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T11:48:13.948Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1490471, "meanTokensPerSec": 25.92, "genTokensPerSec": 25.92, "genTokens": 28044, "promptTokensPerSec": 419.57, "peakRssMb": 47902, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 115230, "output": 8371 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-craftbook-ab-test-readout", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 3 }, "trialId": "craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-11-01T11-48-40-583Z-yu9s", "success": true, "includedInCapability": true, "durationMs": 2103923, "timing": { "startedAt": "2026-11-01T11:48:40.584Z", "finishedAt": "2026-11-01T12:23:44.507Z", "timeToFirstArtifactMs": 40091, "timeToLastArtifactWriteMs": 2093419, "timeToFirstTokenMs": 23024, "firstTurnTtftMs": 16518, "timeToFirstToolCallMs": 34202 }, "metrics": { "process": { "sampleCount": 418, "peakRssMb": 13075, "peakCpuPercent": 160.9, "firstAtMs": 0, "lastAtMs": 2093502 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 418, "peakUsedMb": 68277, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 190309, "totalOutputTokens": 58556, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 182334, "todayTokensCached": 140695, "todayTokensOut": 15742, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 182334, "totalTokensCached": 140695, "totalTokensOut": 15742, "totalCost": 0, "medianOutputTokensPerSec": 36.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.9, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5955, "cachedInputTokens": 1302, "outputTokens": 279, "cost": 0, "durationMs": 12587, "outputTokensPerSec": 46.31284601326877, "at": "2026-11-01T12:23:34.006Z" }, "lastUpdated": "2026-11-01T12:23:34.006Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T12:23:36.962Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2095308, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 58556, "promptTokensPerSec": null, "peakRssMb": 13075, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 182334, "output": 15742 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1175, "sampleCount": 19, "structuredSampleCount": 0, "firstTokenSampleCount": 19, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-productivity-craftbook-annotated-bibliography", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 3 }, "trialId": "craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-11-01T12-24-06-811Z-d41h", "success": true, "includedInCapability": true, "durationMs": 377471, "timing": { "startedAt": "2026-11-01T12:24:06.812Z", "finishedAt": "2026-11-01T12:30:24.283Z", "timeToFirstArtifactMs": 47087, "timeToLastArtifactWriteMs": 365126, "timeToFirstTokenMs": 25645, "firstTurnTtftMs": 19140, "timeToFirstToolCallMs": 26741 }, "metrics": { "process": { "sampleCount": 74, "peakRssMb": 19052, "peakCpuPercent": 139.9, "firstAtMs": 0, "lastAtMs": 366228 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 74, "peakUsedMb": 83826, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44337, "totalOutputTokens": 6603, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44193, "todayTokensCached": 42629, "todayTokensOut": 4709, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44193, "totalTokensCached": 42629, "totalTokensOut": 4709, "totalCost": 0, "medianOutputTokensPerSec": 32.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.1, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12176, "cachedInputTokens": 12121, "outputTokens": 1583, "cost": 0, "durationMs": 128476, "outputTokensPerSec": 24.534075671567063, "at": "2026-11-01T12:30:11.938Z" }, "lastUpdated": "2026-11-01T12:30:11.938Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T12:30:14.635Z" } }, "source": "engine-log" }, "derived": { "durationMs": 366751, "meanTokensPerSec": 31, "genTokensPerSec": 31, "genTokens": 6603, "promptTokensPerSec": null, "peakRssMb": 19052, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 44193, "output": 4709 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 621, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-productivity-craftbook-annotated-bibliography", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 3 }, "trialId": "craftbook-annotated-bibliography-qwen3-8-27b-q4-2026-11-01T12-30-48-552Z-dwt7", "success": true, "includedInCapability": true, "durationMs": 367006, "timing": { "startedAt": "2026-11-01T12:30:48.553Z", "finishedAt": "2026-11-01T12:36:55.559Z", "timeToFirstArtifactMs": 76391, "timeToLastArtifactWriteMs": 358117, "timeToFirstTokenMs": 31310, "firstTurnTtftMs": 24573, "timeToFirstToolCallMs": 39979 }, "metrics": { "process": { "sampleCount": 73, "peakRssMb": 44668, "peakCpuPercent": 138.2, "firstAtMs": 0, "lastAtMs": 361189 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 73, "peakUsedMb": 95723, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44723, "totalOutputTokens": 6698, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44937, "todayTokensCached": 43021, "todayTokensOut": 4205, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44937, "totalTokensCached": 43021, "totalTokensOut": 4205, "totalCost": 0, "medianOutputTokensPerSec": 29.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.4, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12282, "cachedInputTokens": 12212, "outputTokens": 384, "cost": 0, "durationMs": 71499, "outputTokensPerSec": 24.090827577795604, "at": "2026-11-01T12:36:46.670Z" }, "lastUpdated": "2026-11-01T12:36:46.670Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T12:36:51.318Z" } }, "source": "engine-log" }, "derived": { "durationMs": 361712, "meanTokensPerSec": 30.3, "genTokensPerSec": 30.3, "genTokens": 6698, "promptTokensPerSec": 432.34, "peakRssMb": 44668, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44937, "output": 4205 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-craftbook-annotated-bibliography", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 3 }, "trialId": "craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-11-01T12-37-17-075Z-ced9", "success": true, "includedInCapability": true, "durationMs": 663373, "timing": { "startedAt": "2026-11-01T12:37:17.076Z", "finishedAt": "2026-11-01T12:48:20.449Z", "timeToFirstArtifactMs": 60087, "timeToLastArtifactWriteMs": 656647, "timeToFirstTokenMs": 20209, "firstTurnTtftMs": 13697, "timeToFirstToolCallMs": 51066 }, "metrics": { "process": { "sampleCount": 132, "peakRssMb": 12922, "peakCpuPercent": 142.5, "firstAtMs": 0, "lastAtMs": 657390 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 132, "peakUsedMb": 66319, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 84580, "totalOutputTokens": 17061, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 87691, "todayTokensCached": 84507, "todayTokensOut": 3836, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 87691, "totalTokensCached": 84507, "totalTokensOut": 3836, "totalCost": 0, "medianOutputTokensPerSec": 37.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.4, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11045, "cachedInputTokens": 10225, "outputTokens": 1207, "cost": 0, "durationMs": 55001, "outputTokensPerSec": 35.055103930812905, "at": "2026-11-01T12:48:13.723Z" }, "lastUpdated": "2026-11-01T12:48:13.723Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T12:48:16.072Z" } }, "source": "engine-log" }, "derived": { "durationMs": 657929, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 17061, "promptTokensPerSec": null, "peakRssMb": 12922, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 87691, "output": 3836 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1276, "sampleCount": 8, "structuredSampleCount": 0, "firstTokenSampleCount": 8, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-craftbook-annotated-bibliography", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 3 }, "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-11-01T12-48-41-572Z-nrnh", "success": true, "includedInCapability": true, "durationMs": 134368, "timing": { "startedAt": "2026-11-01T12:48:41.572Z", "finishedAt": "2026-11-01T12:50:55.940Z", "timeToFirstArtifactMs": 30657, "timeToLastArtifactWriteMs": 121751, "timeToFirstTokenMs": 18157, "firstTurnTtftMs": 11432, "timeToFirstToolCallMs": 25647 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 26088, "peakCpuPercent": 119.3, "firstAtMs": 0, "lastAtMs": 125403 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 78280, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 41889, "totalOutputTokens": 4290, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 41864, "todayTokensCached": 40648, "todayTokensOut": 1338, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 41864, "totalTokensCached": 40648, "totalTokensOut": 1338, "totalCost": 0, "medianOutputTokensPerSec": 74.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 74.8, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11068, "cachedInputTokens": 10751, "outputTokens": 120, "cost": 0, "durationMs": 25256, "outputTokensPerSec": 70.73380077082011, "at": "2026-11-01T12:50:43.323Z" }, "lastUpdated": "2026-11-01T12:50:43.323Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T12:50:48.550Z" } }, "source": "engine-log" }, "derived": { "durationMs": 125928, "meanTokensPerSec": 76.23, "genTokensPerSec": 76.23, "genTokens": 4290, "promptTokensPerSec": 1134.76, "peakRssMb": 26088, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 41864, "output": 1338 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-productivity-craftbook-research-to-document", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 3 }, "trialId": "craftbook-research-to-document-qwen3-8-27b-q4-2026-11-01T12-51-19-395Z-qx5u", "success": true, "includedInCapability": true, "durationMs": 645928, "timing": { "startedAt": "2026-11-01T12:51:19.395Z", "finishedAt": "2026-11-01T13:02:05.323Z", "timeToFirstArtifactMs": 57786, "timeToLastArtifactWriteMs": 197209, "timeToFirstTokenMs": 29436, "firstTurnTtftMs": 22718, "timeToFirstToolCallMs": 37008 }, "metrics": { "process": { "sampleCount": 128, "peakRssMb": 46996, "peakCpuPercent": 141.5, "firstAtMs": 0, "lastAtMs": 637083 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 128, "peakUsedMb": 98543, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 132623, "totalOutputTokens": 6468, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 101915, "todayTokensCached": 100159, "todayTokensOut": 1797, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 101915, "totalTokensCached": 100159, "totalTokensOut": 1797, "totalCost": 0, "medianOutputTokensPerSec": 33.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 33.3, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 35637, "cachedInputTokens": 35024, "outputTokens": 72, "cost": 0, "durationMs": 204623, "outputTokensPerSec": 31.446903011550315, "at": "2026-11-01T13:01:56.305Z" }, "lastUpdated": "2026-11-01T13:01:56.305Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:01:58.074Z" } }, "source": "engine-log" }, "derived": { "durationMs": 637628, "meanTokensPerSec": 28.81, "genTokensPerSec": 28.81, "genTokens": 6468, "promptTokensPerSec": 351.41, "peakRssMb": 46996, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 101915, "output": 1797 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-productivity-craftbook-research-to-document", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 3 }, "trialId": "craftbook-research-to-document-mlx-gemma4-12b-q4-2026-11-01T13-02-27-242Z-hrqi", "success": true, "includedInCapability": true, "durationMs": 743251, "timing": { "startedAt": "2026-11-01T13:02:27.242Z", "finishedAt": "2026-11-01T13:14:50.493Z", "timeToFirstArtifactMs": 33638, "timeToLastArtifactWriteMs": 102255, "timeToFirstTokenMs": 20294, "firstTurnTtftMs": 13733, "timeToFirstToolCallMs": 25549 }, "metrics": { "process": { "sampleCount": 148, "peakRssMb": 11980, "peakCpuPercent": 120.4, "firstAtMs": 0, "lastAtMs": 737663 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 148, "peakUsedMb": 93428, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 109161, "totalOutputTokens": 12448, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 72955, "todayTokensCached": 71763, "todayTokensOut": 601, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 72955, "totalTokensCached": 71763, "totalTokensOut": 601, "totalCost": 0, "medianOutputTokensPerSec": 38.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.2, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16053, "cachedInputTokens": 15671, "outputTokens": 35, "cost": 0, "durationMs": 38756, "outputTokensPerSec": 40.22539233929807, "at": "2026-11-01T13:14:43.376Z" }, "lastUpdated": "2026-11-01T13:14:43.376Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:14:46.744Z" } }, "source": "engine-log" }, "derived": { "durationMs": 738439, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 12448, "promptTokensPerSec": null, "peakRssMb": 11980, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 72955, "output": 601 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1204.5, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-productivity-craftbook-research-to-document", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 3 }, "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-11-01T13-15-13-281Z-67gt", "success": true, "includedInCapability": true, "durationMs": 210824, "timing": { "startedAt": "2026-11-01T13:15:13.281Z", "finishedAt": "2026-11-01T13:18:44.105Z", "timeToFirstArtifactMs": 28631, "timeToLastArtifactWriteMs": 158844, "timeToFirstTokenMs": 18081, "firstTurnTtftMs": 11382, "timeToFirstToolCallMs": 22528 }, "metrics": { "process": { "sampleCount": 41, "peakRssMb": 29209, "peakCpuPercent": 139.8, "firstAtMs": 0, "lastAtMs": 200651 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 41, "peakUsedMb": 78981, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 94330, "totalOutputTokens": 5116, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 71036, "todayTokensCached": 69324, "todayTokensOut": 691, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 71036, "totalTokensCached": 69324, "totalTokensOut": 691, "totalCost": 0, "medianOutputTokensPerSec": 75.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 75.8, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18712, "cachedInputTokens": 18365, "outputTokens": 41, "cost": 0, "durationMs": 36063, "outputTokensPerSec": 75.8236342267885, "at": "2026-11-01T13:18:31.365Z" }, "lastUpdated": "2026-11-01T13:18:31.365Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:18:35.399Z" } }, "source": "engine-log" }, "derived": { "durationMs": 201174, "meanTokensPerSec": 74.06, "genTokensPerSec": 74.06, "genTokens": 5116, "promptTokensPerSec": 1018.74, "peakRssMb": 29209, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 71036, "output": 691 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-productivity-craftbook-research-to-document", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 3 }, "trialId": "craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-11-01T13-19-07-756Z-nrew", "success": true, "includedInCapability": true, "durationMs": 769812, "timing": { "startedAt": "2026-11-01T13:19:07.756Z", "finishedAt": "2026-11-01T13:31:57.568Z", "timeToFirstArtifactMs": 48541, "timeToLastArtifactWriteMs": 214588, "timeToFirstTokenMs": 26762, "firstTurnTtftMs": 20251, "timeToFirstToolCallMs": 27666 }, "metrics": { "process": { "sampleCount": 153, "peakRssMb": 21021, "peakCpuPercent": 135.9, "firstAtMs": 0, "lastAtMs": 762519 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 153, "peakUsedMb": 88831, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 119747, "totalOutputTokens": 4732, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 91384, "todayTokensCached": 86799, "todayTokensOut": 1984, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 91384, "totalTokensCached": 86799, "totalTokensOut": 1984, "totalCost": 0, "medianOutputTokensPerSec": 28.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.3, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 30687, "cachedInputTokens": 26652, "outputTokens": 73, "cost": 0, "durationMs": 220052, "outputTokensPerSec": 25.668500672316558, "at": "2026-11-01T13:31:50.183Z" }, "lastUpdated": "2026-11-01T13:31:50.183Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:31:52.142Z" } }, "source": "engine-log" }, "derived": { "durationMs": 763317, "meanTokensPerSec": 29, "genTokensPerSec": 29, "genTokens": 4732, "promptTokensPerSec": null, "peakRssMb": 21021, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 91384, "output": 1984 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 555, "sampleCount": 5, "structuredSampleCount": 0, "firstTokenSampleCount": 5, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-developer-fix-squisq-bugs", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 3 }, "trialId": "fix-squisq-bugs-mlx-gemma4-12b-q4-2026-11-01T13-32-19-217Z-9ht8", "success": true, "includedInCapability": true, "durationMs": 705735, "timing": { "startedAt": "2026-11-01T13:32:19.217Z", "finishedAt": "2026-11-01T13:44:04.952Z", "timeToFirstArtifactMs": 56507, "timeToLastArtifactWriteMs": 701334, "timeToFirstTokenMs": 11508, "firstTurnTtftMs": 9873, "timeToFirstToolCallMs": 21657 }, "metrics": { "process": { "sampleCount": 141, "peakRssMb": 10639, "peakCpuPercent": 120.1, "firstAtMs": 0, "lastAtMs": 702508 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 141, "peakUsedMb": 66615, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 314933, "totalOutputTokens": 11837, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 174592, "todayTokensCached": 18885, "todayTokensOut": 6291, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 174592, "totalTokensCached": 18885, "totalTokensOut": 6291, "totalCost": 0, "medianOutputTokensPerSec": 34.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.6, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 34739, "cachedInputTokens": 1303, "outputTokens": 686, "cost": 0, "durationMs": 157996, "outputTokensPerSec": 34.388503278472754, "at": "2026-11-01T13:44:00.551Z" }, "lastUpdated": "2026-11-01T13:44:00.551Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:44:03.584Z" } }, "source": "engine-log" }, "derived": { "durationMs": 703303, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 11837, "promptTokensPerSec": null, "peakRssMb": 10639, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 174592, "output": 6291 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1002, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-fix-squisq-bugs", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 3 }, "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-11-01T13-44-27-329Z-n506", "success": true, "includedInCapability": true, "durationMs": 82422, "timing": { "startedAt": "2026-11-01T13:44:27.330Z", "finishedAt": "2026-11-01T13:45:49.752Z", "timeToFirstArtifactMs": 24716, "timeToLastArtifactWriteMs": 73794, "timeToFirstTokenMs": 10151, "firstTurnTtftMs": 8309, "timeToFirstToolCallMs": 17971 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 20688, "peakCpuPercent": 108.3, "firstAtMs": 0, "lastAtMs": 75245 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 66596, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13991, "totalOutputTokens": 4357, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:45:44.648Z" } }, "source": "engine-log" }, "derived": { "durationMs": 76269, "meanTokensPerSec": 84.19, "genTokensPerSec": 84.19, "genTokens": 4357, "promptTokensPerSec": 866.67, "peakRssMb": 20688, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-fix-squisq-bugs", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 3 }, "trialId": "fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-11-01T13-46-11-565Z-xs5u", "success": true, "includedInCapability": true, "durationMs": 119708, "timing": { "startedAt": "2026-11-01T13:46:11.566Z", "finishedAt": "2026-11-01T13:48:11.274Z", "timeToFirstArtifactMs": 87974, "timeToLastArtifactWriteMs": 108852, "timeToFirstTokenMs": 14571, "firstTurnTtftMs": 12916, "timeToFirstToolCallMs": 17672 }, "metrics": { "process": { "sampleCount": 23, "peakRssMb": 17998, "peakCpuPercent": 121.6, "firstAtMs": 0, "lastAtMs": 110364 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 82832, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 7978, "totalOutputTokens": 2848, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:48:03.776Z" } }, "source": "engine-log" }, "derived": { "durationMs": 111139, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 2848, "promptTokensPerSec": null, "peakRssMb": 17998, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 608, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-fix-squisq-bugs", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 3 }, "trialId": "fix-squisq-bugs-qwen3-8-27b-q4-2026-11-01T13-48-33-915Z-otvg", "success": true, "includedInCapability": true, "durationMs": 120860, "timing": { "startedAt": "2026-11-01T13:48:33.916Z", "finishedAt": "2026-11-01T13:50:34.776Z", "timeToFirstArtifactMs": 68580, "timeToLastArtifactWriteMs": 113414, "timeToFirstTokenMs": 16392, "firstTurnTtftMs": 14563, "timeToFirstToolCallMs": 18934 }, "metrics": { "process": { "sampleCount": 24, "peakRssMb": 38019, "peakCpuPercent": 89.1, "firstAtMs": 0, "lastAtMs": 115383 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 85858, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 9930, "totalOutputTokens": 3089, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:50:31.375Z" } }, "source": "engine-log" }, "derived": { "durationMs": 116408, "meanTokensPerSec": 37.56, "genTokensPerSec": 37.56, "genTokens": 3089, "promptTokensPerSec": 440.23, "peakRssMb": 38019, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-developer-dev-craftbook-routing", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 3 }, "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-11-01T13-50-57-164Z-4yvo", "success": false, "includedInCapability": true, "durationMs": 432124, "timing": { "startedAt": "2026-11-01T13:50:57.165Z", "finishedAt": "2026-11-01T13:58:09.289Z", "timeToFirstArtifactMs": 110319, "timeToLastArtifactWriteMs": 426104, "timeToFirstTokenMs": 15364, "firstTurnTtftMs": 13444, "timeToFirstToolCallMs": 24451 }, "metrics": { "process": { "sampleCount": 86, "peakRssMb": 27507, "peakCpuPercent": 119, "firstAtMs": 0, "lastAtMs": 426390 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 86, "peakUsedMb": 73788, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 175415, "totalOutputTokens": 11688, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 149062, "todayTokensCached": 86171, "todayTokensOut": 2822, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 149062, "totalTokensCached": 86171, "totalTokensOut": 2822, "totalCost": 0, "medianOutputTokensPerSec": 56.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 56.4, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 20303, "outputTokens": 123, "cost": 0, "durationMs": 62554, "outputTokensPerSec": 71.73377596600994, "at": "2026-11-01T13:58:03.269Z" }, "lastUpdated": "2026-11-01T13:58:03.269Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T13:58:06.398Z" } }, "source": "engine-log" }, "derived": { "durationMs": 428183, "meanTokensPerSec": 58.38, "genTokensPerSec": 58.38, "genTokens": 11688, "promptTokensPerSec": 856.11, "peakRssMb": 27507, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 149062, "output": 2822 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-dev-craftbook-routing", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 3 }, "trialId": "dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-11-01T13-58-35-221Z-tw5v", "success": true, "includedInCapability": true, "durationMs": 1061929, "timing": { "startedAt": "2026-11-01T13:58:35.222Z", "finishedAt": "2026-11-01T14:16:17.151Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 30451, "firstTurnTtftMs": 28747, "timeToFirstToolCallMs": 70737 }, "metrics": { "process": { "sampleCount": 211, "peakRssMb": 19107, "peakCpuPercent": 119, "firstAtMs": 0, "lastAtMs": 1055023 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 211, "peakUsedMb": 83649, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 25656, "totalOutputTokens": 14276, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:16:15.603Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1059327, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 14276, "promptTokensPerSec": null, "peakRssMb": 19107, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 676.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-dev-craftbook-routing", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 3 }, "trialId": "dev-craftbook-routing-qwen3-8-27b-q4-2026-11-01T14-16-42-152Z-gyd5", "success": false, "includedInCapability": true, "durationMs": 1124505, "timing": { "startedAt": "2026-11-01T14:16:42.154Z", "finishedAt": "2026-11-01T14:35:26.659Z", "timeToFirstArtifactMs": 449732, "timeToLastArtifactWriteMs": 1093737, "timeToFirstTokenMs": 28942, "firstTurnTtftMs": 27049, "timeToFirstToolCallMs": 51290 }, "metrics": { "process": { "sampleCount": 223, "peakRssMb": 49276, "peakCpuPercent": 119, "firstAtMs": 1, "lastAtMs": 1113583 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 223, "peakUsedMb": 98134, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 254581, "totalOutputTokens": 7855, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 123567, "todayTokensCached": 83449, "todayTokensOut": 2500, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 123567, "totalTokensCached": 83449, "totalTokensOut": 2500, "totalCost": 0, "medianOutputTokensPerSec": 29.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.8, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 30337, "cachedInputTokens": 30082, "outputTokens": 371, "cost": 0, "durationMs": 157756, "outputTokensPerSec": 19.148562778744207, "at": "2026-11-01T14:35:16.350Z" }, "lastUpdated": "2026-11-01T14:35:16.350Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:35:20.848Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1117648, "meanTokensPerSec": 26.98, "genTokensPerSec": 26.98, "genTokens": 7855, "promptTokensPerSec": 322.78, "peakRssMb": 49276, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 123567, "output": 2500 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-dev-craftbook-routing", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 3 }, "trialId": "dev-craftbook-routing-mlx-gemma4-12b-q4-2026-11-01T14-35-50-333Z-ec97", "success": false, "includedInCapability": true, "durationMs": 501829, "timing": { "startedAt": "2026-11-01T14:35:50.333Z", "finishedAt": "2026-11-01T14:44:12.162Z", "timeToFirstArtifactMs": 384123, "timeToLastArtifactWriteMs": 497752, "timeToFirstTokenMs": 16884, "firstTurnTtftMs": 15274, "timeToFirstToolCallMs": 36637 }, "metrics": { "process": { "sampleCount": 100, "peakRssMb": 11224, "peakCpuPercent": 117.9, "firstAtMs": 0, "lastAtMs": 496820 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 100, "peakUsedMb": 66115, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 31553, "totalOutputTokens": 14811, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 91676, "todayTokensCached": 78867, "todayTokensOut": 12432, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 91676, "totalTokensCached": 78867, "totalTokensOut": 12432, "totalCost": 0, "medianOutputTokensPerSec": 36.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.5, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8872, "cachedInputTokens": 8040, "outputTokens": 276, "cost": 0, "durationMs": 8762, "outputTokensPerSec": 36.5315603702474, "at": "2026-11-01T14:44:08.085Z" }, "lastUpdated": "2026-11-01T14:44:08.085Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:44:10.658Z" } }, "source": "engine-log" }, "derived": { "durationMs": 499362, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 14811, "promptTokensPerSec": null, "peakRssMb": 11224, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 91676, "output": 12432 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 558, "sampleCount": 11, "structuredSampleCount": 0, "firstTokenSampleCount": 11, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-developer-interface-contract", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "interface-contract", "repetition": 3 }, "trialId": "interface-contract-mlx-qwen3-8-27b-q4-2026-11-01T14-44-34-546Z-rt04", "success": true, "includedInCapability": true, "durationMs": 113532, "timing": { "startedAt": "2026-11-01T14:44:34.546Z", "finishedAt": "2026-11-01T14:46:28.078Z", "timeToFirstArtifactMs": 111083, "timeToLastArtifactWriteMs": 111100, "timeToFirstTokenMs": 29120, "firstTurnTtftMs": 27420, "timeToFirstToolCallMs": 43406 }, "metrics": { "process": { "sampleCount": 23, "peakRssMb": 18412, "peakCpuPercent": 115.1, "firstAtMs": 0, "lastAtMs": 110362 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 82957, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18568, "totalOutputTokens": 1922, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12326, "todayTokensCached": 12064, "todayTokensOut": 152, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12326, "totalTokensCached": 12064, "totalTokensOut": 152, "totalCost": 0, "medianOutputTokensPerSec": 28.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.6, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12326, "cachedInputTokens": 12064, "outputTokens": 152, "cost": 0, "durationMs": 63452, "outputTokensPerSec": 28.56782132991633, "at": "2026-11-01T14:45:39.698Z" }, "lastUpdated": "2026-11-01T14:45:39.698Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:46:26.500Z" } }, "source": "engine-log" }, "derived": { "durationMs": 110886, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 1922, "promptTokensPerSec": null, "peakRssMb": 18412, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 12326, "output": 152 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 581, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-interface-contract", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "interface-contract", "repetition": 3 }, "trialId": "interface-contract-qwen3-8-27b-q4-2026-11-01T14-46-50-483Z-stwl", "success": true, "includedInCapability": true, "durationMs": 150440, "timing": { "startedAt": "2026-11-01T14:46:50.484Z", "finishedAt": "2026-11-01T14:49:20.924Z", "timeToFirstArtifactMs": 127652, "timeToLastArtifactWriteMs": 142644, "timeToFirstTokenMs": 23495, "firstTurnTtftMs": 21614, "timeToFirstToolCallMs": 40482 }, "metrics": { "process": { "sampleCount": 30, "peakRssMb": 40573, "peakCpuPercent": 118.5, "firstAtMs": 0, "lastAtMs": 145465 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 92284, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 16977, "totalOutputTokens": 2905, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 24266, "todayTokensCached": 22493, "todayTokensOut": 596, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 24266, "totalTokensCached": 22493, "totalTokensOut": 596, "totalCost": 0, "medianOutputTokensPerSec": 31.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.1, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8628, "cachedInputTokens": 8299, "outputTokens": 178, "cost": 0, "durationMs": 39443, "outputTokensPerSec": 27.83729681918779, "at": "2026-11-01T14:48:27.535Z" }, "lastUpdated": "2026-11-01T14:48:27.535Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:49:17.520Z" } }, "source": "engine-log" }, "derived": { "durationMs": 145988, "meanTokensPerSec": 32.58, "genTokensPerSec": 32.58, "genTokens": 2905, "promptTokensPerSec": 457.29, "peakRssMb": 40573, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 24266, "output": 596 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-interface-contract", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "interface-contract", "repetition": 3 }, "trialId": "interface-contract-mlx-gemma4-12b-q4-2026-11-01T14-49-42-350Z-poi8", "success": true, "includedInCapability": true, "durationMs": 77999, "timing": { "startedAt": "2026-11-01T14:49:42.351Z", "finishedAt": "2026-11-01T14:51:00.350Z", "timeToFirstArtifactMs": 62844, "timeToLastArtifactWriteMs": 76027, "timeToFirstTokenMs": 16958, "firstTurnTtftMs": 15277, "timeToFirstToolCallMs": 40756 }, "metrics": { "process": { "sampleCount": 16, "peakRssMb": 10642, "peakCpuPercent": 122.9, "firstAtMs": 0, "lastAtMs": 75259 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 66628, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 18677, "totalOutputTokens": 983, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 10584, "todayTokensCached": 10409, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 10584, "totalTokensCached": 10409, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 41.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 41.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10584, "cachedInputTokens": 10409, "outputTokens": 23, "cost": 0, "durationMs": 40796, "outputTokensPerSec": 41.90519753071533, "at": "2026-11-01T14:50:24.828Z" }, "lastUpdated": "2026-11-01T14:50:24.828Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:50:59.217Z" } }, "source": "engine-log" }, "derived": { "durationMs": 75800, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 983, "promptTokensPerSec": null, "peakRssMb": 10642, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 10584, "output": 23 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1300.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-interface-contract", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "interface-contract", "repetition": 3 }, "trialId": "interface-contract-gemma4-12b-q4-2026-11-01T14-51-22-142Z-bss4", "success": true, "includedInCapability": true, "durationMs": 55908, "timing": { "startedAt": "2026-11-01T14:51:22.143Z", "finishedAt": "2026-11-01T14:52:18.051Z", "timeToFirstArtifactMs": 38231, "timeToLastArtifactWriteMs": 46010, "timeToFirstTokenMs": 11578, "firstTurnTtftMs": 9704, "timeToFirstToolCallMs": 21432 }, "metrics": { "process": { "sampleCount": 10, "peakRssMb": 20590, "peakCpuPercent": 114.6, "firstAtMs": 1, "lastAtMs": 45142 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 72741, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 15340, "totalOutputTokens": 1628, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7326, "todayTokensCached": 7155, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7326, "totalTokensCached": 7155, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 69, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 69, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7326, "cachedInputTokens": 7155, "outputTokens": 23, "cost": 0, "durationMs": 21192, "outputTokensPerSec": 69.01636001443069, "at": "2026-11-01T14:51:45.209Z" }, "lastUpdated": "2026-11-01T14:51:45.209Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:52:08.870Z" } }, "source": "engine-log" }, "derived": { "durationMs": 45681, "meanTokensPerSec": 89.88, "genTokensPerSec": 89.88, "genTokens": 1628, "promptTokensPerSec": 1156.3, "peakRssMb": 20590, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7326, "output": 23 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-developer-craftbook-code-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 3 }, "trialId": "craftbook-code-review-qwen3-8-27b-q4-2026-11-01T14-52-38-908Z-m7u5", "success": true, "includedInCapability": true, "durationMs": 230670, "timing": { "startedAt": "2026-11-01T14:52:38.908Z", "finishedAt": "2026-11-01T14:56:29.578Z", "timeToFirstArtifactMs": 143343, "timeToLastArtifactWriteMs": 143343, "timeToFirstTokenMs": 27282, "firstTurnTtftMs": 20649, "timeToFirstToolCallMs": 28281 }, "metrics": { "process": { "sampleCount": 46, "peakRssMb": 42246, "peakCpuPercent": 113.6, "firstAtMs": 1, "lastAtMs": 225733 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 93322, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 47652, "totalOutputTokens": 3194, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 33813, "todayTokensCached": 21655, "todayTokensOut": 1695, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 33813, "totalTokensCached": 21655, "totalTokensOut": 1695, "totalCost": 0, "medianOutputTokensPerSec": 38.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 38.9, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11917, "cachedInputTokens": 11602, "outputTokens": 60, "cost": 0, "durationMs": 42002, "outputTokensPerSec": 38.929992656087826, "at": "2026-11-01T14:55:47.499Z" }, "lastUpdated": "2026-11-01T14:55:47.499Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:56:26.218Z" } }, "source": "engine-log" }, "derived": { "durationMs": 226260, "meanTokensPerSec": 33.48, "genTokensPerSec": 33.48, "genTokens": 3194, "promptTokensPerSec": 456.68, "peakRssMb": 42246, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 33813, "output": 1695 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-craftbook-code-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 3 }, "trialId": "craftbook-code-review-mlx-gemma4-12b-q4-2026-11-01T14-56-53-006Z-qxih", "success": true, "includedInCapability": true, "durationMs": 158690, "timing": { "startedAt": "2026-11-01T14:56:53.006Z", "finishedAt": "2026-11-01T14:59:31.696Z", "timeToFirstArtifactMs": 121452, "timeToLastArtifactWriteMs": 121452, "timeToFirstTokenMs": 19891, "firstTurnTtftMs": 13326, "timeToFirstToolCallMs": 25364 }, "metrics": { "process": { "sampleCount": 32, "peakRssMb": 10878, "peakCpuPercent": 99.2, "firstAtMs": 0, "lastAtMs": 155567 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 66155, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 36991, "totalOutputTokens": 2803, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 31977, "todayTokensCached": 20842, "todayTokensOut": 2854, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 31977, "totalTokensCached": 20842, "totalTokensOut": 2854, "totalCost": 0, "medianOutputTokensPerSec": 37.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.7, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11330, "cachedInputTokens": 11145, "outputTokens": 33, "cost": 0, "durationMs": 15837, "outputTokensPerSec": 37.704143642539066, "at": "2026-11-01T14:59:15.418Z" }, "lastUpdated": "2026-11-01T14:59:15.418Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T14:59:30.165Z" } }, "source": "engine-log" }, "derived": { "durationMs": 156090, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 2803, "promptTokensPerSec": null, "peakRssMb": 10878, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 31977, "output": 2854 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1239, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-craftbook-code-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 3 }, "trialId": "craftbook-code-review-gemma4-12b-q4-2026-11-01T14-59-55-793Z-7dm4", "success": false, "includedInCapability": true, "durationMs": 1177838, "timing": { "startedAt": "2026-11-01T14:59:55.793Z", "finishedAt": "2026-11-01T15:19:33.631Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 17453, "firstTurnTtftMs": 10716, "timeToFirstToolCallMs": 20484 }, "metrics": { "process": { "sampleCount": 235, "peakRssMb": 25484, "peakCpuPercent": 115.7, "firstAtMs": 0, "lastAtMs": 1176123 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 235, "peakUsedMb": 79871, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 80304, "totalOutputTokens": 21605, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 92440, "todayTokensCached": 75535, "todayTokensOut": 12152, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 92440, "totalTokensCached": 75535, "totalTokensOut": 12152, "totalCost": 0, "medianOutputTokensPerSec": 80.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 80.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14817, "outputTokens": 602, "cost": 0, "durationMs": 17867, "outputTokensPerSec": 80.20123704736677, "at": "2026-11-01T15:14:29.542Z" }, "lastUpdated": "2026-11-01T15:14:29.542Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T15:19:33.509Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1176668, "meanTokensPerSec": 85.05, "genTokensPerSec": 85.05, "genTokens": 21605, "promptTokensPerSec": 1180.38, "peakRssMb": 25484, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 92440, "output": 12152 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-craftbook-code-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 3 }, "trialId": "craftbook-code-review-mlx-qwen3-8-27b-q4-2026-11-01T15-19-55-409Z-5gkp", "success": true, "includedInCapability": true, "durationMs": 200817, "timing": { "startedAt": "2026-11-01T15:19:55.409Z", "finishedAt": "2026-11-01T15:23:16.226Z", "timeToFirstArtifactMs": 157355, "timeToLastArtifactWriteMs": 157355, "timeToFirstTokenMs": 21060, "firstTurnTtftMs": 14571, "timeToFirstToolCallMs": 21944 }, "metrics": { "process": { "sampleCount": 40, "peakRssMb": 18275, "peakCpuPercent": 108.6, "firstAtMs": 0, "lastAtMs": 195697 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 83043, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 31428, "totalOutputTokens": 3849, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 30486, "todayTokensCached": 19658, "todayTokensOut": 2564, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 30486, "totalTokensCached": 19658, "totalTokensOut": 2564, "totalCost": 0, "medianOutputTokensPerSec": 36.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.5, "turns": 3 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 10244, "cachedInputTokens": 10156, "outputTokens": 84, "cost": 0, "durationMs": 20678, "outputTokensPerSec": 36.33512078251726, "at": "2026-11-01T15:22:57.803Z" }, "lastUpdated": "2026-11-01T15:22:57.803Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T15:23:12.702Z" } }, "source": "engine-log" }, "derived": { "durationMs": 196221, "meanTokensPerSec": 37, "genTokensPerSec": 37, "genTokens": 3849, "promptTokensPerSec": null, "peakRssMb": 18275, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 30486, "output": 2564 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 681.5, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-developer-craftbook-api-contract-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 3 }, "trialId": "craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-11-01T15-23-38-905Z-ftx0", "success": true, "includedInCapability": true, "durationMs": 457631, "timing": { "startedAt": "2026-11-01T15:23:38.906Z", "finishedAt": "2026-11-01T15:31:16.537Z", "timeToFirstArtifactMs": 90907, "timeToLastArtifactWriteMs": 415074, "timeToFirstTokenMs": 20651, "firstTurnTtftMs": 14140, "timeToFirstToolCallMs": 34895 }, "metrics": { "process": { "sampleCount": 91, "peakRssMb": 11714, "peakCpuPercent": 148.3, "firstAtMs": 1, "lastAtMs": 451734 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 91, "peakUsedMb": 66872, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 67316, "totalOutputTokens": 9707, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 79546, "todayTokensCached": 75014, "todayTokensOut": 2853, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 79546, "totalTokensCached": 75014, "totalTokensOut": 2853, "totalCost": 0, "medianOutputTokensPerSec": 35.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8099, "cachedInputTokens": 7937, "outputTokens": 35, "cost": 0, "durationMs": 25287, "outputTokensPerSec": 35.243506902174445, "at": "2026-11-01T15:31:00.790Z" }, "lastUpdated": "2026-11-01T15:31:00.790Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T15:31:12.230Z" } }, "source": "engine-log" }, "derived": { "durationMs": 452260, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 9707, "promptTokensPerSec": null, "peakRssMb": 11714, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 79546, "output": 2853 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1099, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-craftbook-api-contract-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 3 }, "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-11-01T15-31-38-658Z-7rrt", "success": true, "includedInCapability": true, "durationMs": 440645, "timing": { "startedAt": "2026-11-01T15:31:38.659Z", "finishedAt": "2026-11-01T15:38:59.304Z", "timeToFirstArtifactMs": 47153, "timeToLastArtifactWriteMs": 402201, "timeToFirstTokenMs": 19124, "firstTurnTtftMs": 12400, "timeToFirstToolCallMs": 24777 }, "metrics": { "process": { "sampleCount": 88, "peakRssMb": 25416, "peakCpuPercent": 185.6, "firstAtMs": 0, "lastAtMs": 436467 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 88, "peakUsedMb": 79159, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 123780, "totalOutputTokens": 16078, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 107180, "todayTokensCached": 90207, "todayTokensOut": 4267, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 107180, "totalTokensCached": 90207, "totalTokensOut": 4267, "totalCost": 0, "medianOutputTokensPerSec": 75.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 75.5, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8027, "cachedInputTokens": 7832, "outputTokens": 117, "cost": 0, "durationMs": 19931, "outputTokensPerSec": 86.17269900321959, "at": "2026-11-01T15:38:45.220Z" }, "lastUpdated": "2026-11-01T15:38:45.220Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T15:38:56.702Z" } }, "source": "engine-log" }, "derived": { "durationMs": 436989, "meanTokensPerSec": 63.64, "genTokensPerSec": 63.64, "genTokens": 16078, "promptTokensPerSec": 999.32, "peakRssMb": 25416, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 107180, "output": 4267 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-craftbook-api-contract-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 3 }, "trialId": "craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-11-01T15-39-24-739Z-zzmy", "success": true, "includedInCapability": true, "durationMs": 1284411, "timing": { "startedAt": "2026-11-01T15:39:24.739Z", "finishedAt": "2026-11-01T16:00:49.150Z", "timeToFirstArtifactMs": 318138, "timeToLastArtifactWriteMs": 1277730, "timeToFirstTokenMs": 29056, "firstTurnTtftMs": 22360, "timeToFirstToolCallMs": 52028 }, "metrics": { "process": { "sampleCount": 256, "peakRssMb": 18497, "peakCpuPercent": 110.8, "firstAtMs": 0, "lastAtMs": 1279276 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 256, "peakUsedMb": 81646, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 76095, "totalOutputTokens": 26094, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 84105, "todayTokensCached": 69426, "todayTokensOut": 16176, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 84105, "totalTokensCached": 69426, "totalTokensOut": 16176, "totalCost": 0, "medianOutputTokensPerSec": 24.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.9, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 26236, "cachedInputTokens": 25397, "outputTokens": 635, "cost": 0, "durationMs": 160543, "outputTokensPerSec": 22.743063962188216, "at": "2026-11-01T16:00:42.469Z" }, "lastUpdated": "2026-11-01T16:00:42.469Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T16:00:45.722Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1279821, "meanTokensPerSec": 25, "genTokensPerSec": 25, "genTokens": 26094, "promptTokensPerSec": null, "peakRssMb": 18497, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 84105, "output": 16176 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 519, "sampleCount": 4, "structuredSampleCount": 0, "firstTokenSampleCount": 4, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-craftbook-api-contract-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 3 }, "trialId": "craftbook-api-contract-review-qwen3-8-27b-q4-2026-11-01T16-01-12-839Z-5foc", "success": true, "includedInCapability": true, "durationMs": 1262187, "timing": { "startedAt": "2026-11-01T16:01:12.840Z", "finishedAt": "2026-11-01T16:22:15.027Z", "timeToFirstArtifactMs": 140175, "timeToLastArtifactWriteMs": 1257975, "timeToFirstTokenMs": 32345, "firstTurnTtftMs": 25641, "timeToFirstToolCallMs": 44132 }, "metrics": { "process": { "sampleCount": 252, "peakRssMb": 44443, "peakCpuPercent": 113.5, "firstAtMs": 1, "lastAtMs": 1259111 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 252, "peakUsedMb": 96264, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 96764, "totalOutputTokens": 24656, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 80025, "todayTokensCached": 68826, "todayTokensOut": 10497, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 80025, "totalTokensCached": 68826, "totalTokensOut": 10497, "totalCost": 0, "medianOutputTokensPerSec": 26.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.7, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 21812, "cachedInputTokens": 21413, "outputTokens": 1093, "cost": 0, "durationMs": 210825, "outputTokensPerSec": 24.89726117815208, "at": "2026-11-01T16:22:10.815Z" }, "lastUpdated": "2026-11-01T16:22:10.815Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T16:22:13.539Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1259656, "meanTokensPerSec": 25.15, "genTokensPerSec": 25.15, "genTokens": 24656, "promptTokensPerSec": 389.26, "peakRssMb": 44443, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 80025, "output": 10497 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-developer-craftbook-deep-security-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 3 }, "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-11-01T16-22-38-459Z-x9ig", "success": false, "includedInCapability": true, "durationMs": 287873, "timing": { "startedAt": "2026-11-01T16:22:38.460Z", "finishedAt": "2026-11-01T16:27:26.333Z", "timeToFirstArtifactMs": 44782, "timeToLastArtifactWriteMs": 275436, "timeToFirstTokenMs": 13591, "firstTurnTtftMs": 11715, "timeToFirstToolCallMs": 15535 }, "metrics": { "process": { "sampleCount": 56, "peakRssMb": 24683, "peakCpuPercent": 118.1, "firstAtMs": 0, "lastAtMs": 275968 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 56, "peakUsedMb": 70641, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 57657, "totalOutputTokens": 13910, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 133986, "todayTokensCached": 115564, "todayTokensOut": 6967, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 133986, "totalTokensCached": 115564, "totalTokensOut": 6967, "totalCost": 0, "medianOutputTokensPerSec": 72.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 72.2, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22057, "cachedInputTokens": 21176, "outputTokens": 34, "cost": 0, "durationMs": 31614, "outputTokensPerSec": 70.89852058420381, "at": "2026-11-01T16:27:13.896Z" }, "lastUpdated": "2026-11-01T16:27:13.896Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T16:27:18.029Z" } }, "source": "engine-log" }, "derived": { "durationMs": 278522, "meanTokensPerSec": 72.01, "genTokensPerSec": 72.01, "genTokens": 13910, "promptTokensPerSec": 946.63, "peakRssMb": 24683, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 133986, "output": 6967 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-craftbook-deep-security-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 3 }, "trialId": "craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-11-01T16-27-47-991Z-iwh8", "success": true, "includedInCapability": true, "durationMs": 852615, "timing": { "startedAt": "2026-11-01T16:27:47.991Z", "finishedAt": "2026-11-01T16:42:00.606Z", "timeToFirstArtifactMs": 128664, "timeToLastArtifactWriteMs": 848715, "timeToFirstTokenMs": 23779, "firstTurnTtftMs": 22135, "timeToFirstToolCallMs": 26825 }, "metrics": { "process": { "sampleCount": 170, "peakRssMb": 19758, "peakCpuPercent": 123.4, "firstAtMs": 1, "lastAtMs": 847788 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 170, "peakUsedMb": 83458, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 140606, "totalOutputTokens": 12893, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 107773, "todayTokensCached": 25057, "todayTokensOut": 3008, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 107773, "totalTokensCached": 25057, "totalTokensOut": 3008, "totalCost": 0, "medianOutputTokensPerSec": 29.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.6, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 16316, "cachedInputTokens": 0, "outputTokens": 1028, "cost": 0, "durationMs": 64434, "outputTokensPerSec": 30.943841357503167, "at": "2026-11-01T16:41:56.706Z" }, "lastUpdated": "2026-11-01T16:41:56.706Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T16:41:58.390Z" } }, "source": "engine-log" }, "derived": { "durationMs": 849333, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 12893, "promptTokensPerSec": null, "peakRssMb": 19758, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 107773, "output": 3008 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 510, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-craftbook-deep-security-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 3 }, "trialId": "craftbook-deep-security-review-qwen3-8-27b-q4-2026-11-01T16-42-23-995Z-ozn2", "success": true, "includedInCapability": true, "durationMs": 345638, "timing": { "startedAt": "2026-11-01T16:42:23.996Z", "finishedAt": "2026-11-01T16:48:09.634Z", "timeToFirstArtifactMs": 141273, "timeToLastArtifactWriteMs": 336411, "timeToFirstTokenMs": 27767, "firstTurnTtftMs": 25928, "timeToFirstToolCallMs": 31513 }, "metrics": { "process": { "sampleCount": 68, "peakRssMb": 45837, "peakCpuPercent": 99.3, "firstAtMs": 0, "lastAtMs": 336112 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 68, "peakUsedMb": 96515, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 51983, "totalOutputTokens": 5821, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 64743, "todayTokensCached": 45489, "todayTokensOut": 2330, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 64743, "totalTokensCached": 45489, "totalTokensOut": 2330, "totalCost": 0, "medianOutputTokensPerSec": 37.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 37.4, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11769, "cachedInputTokens": 11068, "outputTokens": 83, "cost": 0, "durationMs": 6864, "outputTokensPerSec": 37.10078201209305, "at": "2026-11-01T16:48:00.409Z" }, "lastUpdated": "2026-11-01T16:48:00.409Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T16:48:03.961Z" } }, "source": "engine-log" }, "derived": { "durationMs": 338916, "meanTokensPerSec": 31.18, "genTokensPerSec": 31.18, "genTokens": 5821, "promptTokensPerSec": 405.79, "peakRssMb": 45837, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 64743, "output": 2330 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-craftbook-deep-security-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 3 }, "trialId": "craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-11-01T16-48-31-846Z-3yml", "success": false, "includedInCapability": true, "durationMs": 367434, "timing": { "startedAt": "2026-11-01T16:48:31.846Z", "finishedAt": "2026-11-01T16:54:39.280Z", "timeToFirstArtifactMs": 271307, "timeToLastArtifactWriteMs": 333630, "timeToFirstTokenMs": 15323, "firstTurnTtftMs": 13673, "timeToFirstToolCallMs": 19914 }, "metrics": { "process": { "sampleCount": 73, "peakRssMb": 10492, "peakCpuPercent": 114.8, "firstAtMs": 0, "lastAtMs": 361453 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 73, "peakUsedMb": 66678, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 17803, "totalOutputTokens": 11256, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 19694, "todayTokensCached": 19201, "todayTokensOut": 8414, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 19694, "totalTokensCached": 19201, "totalTokensOut": 8414, "totalCost": 0, "medianOutputTokensPerSec": 35.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10590, "cachedInputTokens": 10146, "outputTokens": 222, "cost": 0, "durationMs": 68766, "outputTokensPerSec": 34.53167647146176, "at": "2026-11-01T16:53:43.711Z" }, "lastUpdated": "2026-11-01T16:53:43.711Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T16:54:37.908Z" } }, "source": "engine-log" }, "derived": { "durationMs": 364999, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 11256, "promptTokensPerSec": null, "peakRssMb": 10492, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 19694, "output": 8414 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 231, "sampleCount": 3, "structuredSampleCount": 0, "firstTokenSampleCount": 3, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-developer-large-pr-review", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "large-pr-review", "repetition": 3 }, "trialId": "large-pr-review-mlx-qwen3-8-27b-q4-2026-11-01T16-55-00-774Z-vl44", "success": true, "includedInCapability": true, "durationMs": 1808901, "timing": { "startedAt": "2026-11-01T16:55:00.774Z", "finishedAt": "2026-11-01T17:25:09.675Z", "timeToFirstArtifactMs": 1783922, "timeToLastArtifactWriteMs": 1799926, "timeToFirstTokenMs": 35152, "firstTurnTtftMs": 33374, "timeToFirstToolCallMs": 37963 }, "metrics": { "process": { "sampleCount": 360, "peakRssMb": 18043, "peakCpuPercent": 124.2, "firstAtMs": 0, "lastAtMs": 1801020 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 360, "peakUsedMb": 84939, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 116309, "totalOutputTokens": 24992, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 115360, "todayTokensCached": 115299, "todayTokensOut": 494, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 115360, "totalTokensCached": 115299, "totalTokensOut": 494, "totalCost": 0, "medianOutputTokensPerSec": 13, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 13, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 115360, "cachedInputTokens": 115299, "outputTokens": 494, "cost": 0, "durationMs": 1782143, "outputTokensPerSec": 13.021678691301782, "at": "2026-11-01T17:24:44.695Z" }, "lastUpdated": "2026-11-01T17:24:44.695Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T17:25:03.388Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1801545, "meanTokensPerSec": 17, "genTokensPerSec": 17, "genTokens": 24992, "promptTokensPerSec": null, "peakRssMb": 18043, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 115360, "output": 494 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 337, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-large-pr-review", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "large-pr-review", "repetition": 3 }, "trialId": "large-pr-review-qwen3-8-27b-q4-2026-11-01T17-25-35-030Z-kbkl", "success": true, "includedInCapability": true, "durationMs": 2084100, "timing": { "startedAt": "2026-11-01T17:25:35.031Z", "finishedAt": "2026-11-01T18:00:19.131Z", "timeToFirstArtifactMs": 1241585, "timeToLastArtifactWriteMs": 2074775, "timeToFirstTokenMs": 40543, "firstTurnTtftMs": 38607, "timeToFirstToolCallMs": 42839 }, "metrics": { "process": { "sampleCount": 415, "peakRssMb": 40307, "peakCpuPercent": 90.8, "firstAtMs": 0, "lastAtMs": 2076756 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 415, "peakUsedMb": 87612, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 187895, "totalOutputTokens": 27691, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 193943, "todayTokensCached": 193002, "todayTokensOut": 828, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 193943, "totalTokensCached": 193002, "totalTokensOut": 828, "totalCost": 0, "medianOutputTokensPerSec": 15.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 15.1, "turns": 2 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 100708, "cachedInputTokens": 100548, "outputTokens": 220, "cost": 0, "durationMs": 722158, "outputTokensPerSec": 15.421439420255277, "at": "2026-11-01T17:58:18.795Z" }, "lastUpdated": "2026-11-01T17:58:18.795Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T18:00:13.363Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2077280, "meanTokensPerSec": 21.72, "genTokensPerSec": 21.72, "genTokens": 27691, "promptTokensPerSec": 237.83, "peakRssMb": 40307, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 193943, "output": 828 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-large-pr-review", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "large-pr-review", "repetition": 3 }, "trialId": "large-pr-review-mlx-gemma4-12b-q4-2026-11-01T18-00-41-204Z-zgpc", "success": false, "includedInCapability": true, "durationMs": 4674450, "timing": { "startedAt": "2026-11-01T18:00:41.205Z", "finishedAt": "2026-11-01T19:18:35.655Z", "timeToFirstArtifactMs": 513703, "timeToLastArtifactWriteMs": 4202154, "timeToFirstTokenMs": 19347, "firstTurnTtftMs": 17549, "timeToFirstToolCallMs": 33697 }, "metrics": { "process": { "sampleCount": 931, "peakRssMb": 12958, "peakCpuPercent": 117.2, "firstAtMs": 0, "lastAtMs": 4669293 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 931, "peakUsedMb": 70584, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 902896, "totalOutputTokens": 73949, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 1107843, "todayTokensCached": 705713, "todayTokensOut": 32628, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 1107843, "totalTokensCached": 705713, "totalTokensOut": 32628, "totalCost": 0, "medianOutputTokensPerSec": 31.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 31.6, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 120734, "cachedInputTokens": 1269, "outputTokens": 5224, "cost": 0, "durationMs": 403503, "outputTokensPerSec": 27.988992578038534, "at": "2026-11-01T19:10:43.359Z" }, "lastUpdated": "2026-11-01T19:10:43.359Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T19:18:32.096Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4669830, "meanTokensPerSec": 30, "genTokensPerSec": 30, "genTokens": 73949, "promptTokensPerSec": null, "peakRssMb": 12958, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 1107843, "output": 32628 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 638, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-large-pr-review", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "large-pr-review", "repetition": 3 }, "trialId": "large-pr-review-gemma4-12b-q4-2026-11-01T19-19-01-384Z-i7ws", "success": false, "includedInCapability": true, "durationMs": 4564871, "timing": { "startedAt": "2026-11-01T19:19:01.384Z", "finishedAt": "2026-11-01T20:35:06.255Z", "timeToFirstArtifactMs": 34764, "timeToLastArtifactWriteMs": 3468146, "timeToFirstTokenMs": 17114, "firstTurnTtftMs": 15133, "timeToFirstToolCallMs": 20632 }, "metrics": { "process": { "sampleCount": 909, "peakRssMb": 26377, "peakCpuPercent": 93.9, "firstAtMs": 0, "lastAtMs": 4554526 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 909, "peakUsedMb": 72700, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 1115046, "totalOutputTokens": 71268, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 824298, "todayTokensCached": 664814, "todayTokensOut": 12815, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 824298, "totalTokensCached": 664814, "totalTokensOut": 12815, "totalCost": 0, "medianOutputTokensPerSec": 47.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 47.3, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 141632, "outputTokens": 3370, "cost": 0, "durationMs": 442374, "outputTokensPerSec": 24.280537722020707, "at": "2026-11-01T20:16:49.530Z" }, "lastUpdated": "2026-11-01T20:16:49.530Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T20:34:57.485Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4555051, "meanTokensPerSec": 38.28, "genTokensPerSec": 38.28, "genTokens": 71268, "promptTokensPerSec": 502.36, "peakRssMb": 26377, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 824298, "output": 12815 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-developer-craftbook-codemod-sweep", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 3 }, "trialId": "craftbook-codemod-sweep-qwen3-8-27b-q4-2026-11-01T20-35-28-695Z-173m", "success": false, "includedInCapability": true, "durationMs": 3385543, "timing": { "startedAt": "2026-11-01T20:35:28.696Z", "finishedAt": "2026-11-01T21:31:54.239Z", "timeToFirstArtifactMs": 198687, "timeToLastArtifactWriteMs": 3328560, "timeToFirstTokenMs": 32606, "firstTurnTtftMs": 25930, "timeToFirstToolCallMs": 37649 }, "metrics": { "process": { "sampleCount": 674, "peakRssMb": 49227, "peakCpuPercent": 142.7, "firstAtMs": 0, "lastAtMs": 3375962 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 674, "peakUsedMb": 97126, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 89666, "totalOutputTokens": 44308, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 47495, "todayTokensCached": 45358, "todayTokensOut": 5096, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 47495, "totalTokensCached": 45358, "totalTokensOut": 5096, "totalCost": 0, "medianOutputTokensPerSec": 26.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.1, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 12548, "cachedInputTokens": 12377, "outputTokens": 1713, "cost": 0, "durationMs": 584290, "outputTokensPerSec": 22.861087539323876, "at": "2026-11-01T20:51:54.993Z" }, "lastUpdated": "2026-11-01T20:51:54.993Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T21:31:46.232Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3376485, "meanTokensPerSec": 22.56, "genTokensPerSec": 22.56, "genTokens": 44308, "promptTokensPerSec": 343.88, "peakRssMb": 49227, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 47495, "output": 5096 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-craftbook-codemod-sweep", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 3 }, "trialId": "craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-11-01T21-32-17-703Z-gd59", "success": false, "includedInCapability": true, "durationMs": 1750436, "timing": { "startedAt": "2026-11-01T21:32:17.703Z", "finishedAt": "2026-11-01T22:01:28.139Z", "timeToFirstArtifactMs": 40736, "timeToLastArtifactWriteMs": 1402753, "timeToFirstTokenMs": 20104, "firstTurnTtftMs": 13585, "timeToFirstToolCallMs": 24383 }, "metrics": { "process": { "sampleCount": 349, "peakRssMb": 11219, "peakCpuPercent": 125, "firstAtMs": 0, "lastAtMs": 1746580 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 349, "peakUsedMb": 66611, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 299243, "totalOutputTokens": 40010, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 199939, "todayTokensCached": 168011, "todayTokensOut": 3622, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 199939, "totalTokensCached": 168011, "totalTokensOut": 3622, "totalCost": 0, "medianOutputTokensPerSec": 34.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 34.3, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 35771, "cachedInputTokens": 35509, "outputTokens": 667, "cost": 0, "durationMs": 59961, "outputTokensPerSec": 33.803836892585856, "at": "2026-11-01T21:50:24.892Z" }, "lastUpdated": "2026-11-01T21:50:24.892Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T22:01:25.892Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1747129, "meanTokensPerSec": 34, "genTokensPerSec": 34, "genTokens": 40010, "promptTokensPerSec": null, "peakRssMb": 11219, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 199939, "output": 3622 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 896, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-craftbook-codemod-sweep", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 3 }, "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-11-01T22-01-50-483Z-a2m8", "success": false, "includedInCapability": true, "durationMs": 1586770, "timing": { "startedAt": "2026-11-01T22:01:50.483Z", "finishedAt": "2026-11-01T22:28:17.253Z", "timeToFirstArtifactMs": 32629, "timeToLastArtifactWriteMs": 1533677, "timeToFirstTokenMs": 18195, "firstTurnTtftMs": 11478, "timeToFirstToolCallMs": 22320 }, "metrics": { "process": { "sampleCount": 315, "peakRssMb": 31203, "peakCpuPercent": 135.3, "firstAtMs": 1, "lastAtMs": 1576073 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 315, "peakUsedMb": 87274, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 384113, "totalOutputTokens": 38126, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 19, "todayTokensIn": 356656, "todayTokensCached": 324288, "todayTokensOut": 9099, "todayCost": 0, "totalTurns": 19, "totalTokensIn": 356656, "totalTokensCached": 324288, "totalTokensOut": 9099, "totalCost": 0, "medianOutputTokensPerSec": 72.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 72.9, "turns": 19 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 25585, "cachedInputTokens": 25369, "outputTokens": 189, "cost": 0, "durationMs": 44842, "outputTokensPerSec": 99.60739849847145, "at": "2026-11-01T22:27:24.160Z" }, "lastUpdated": "2026-11-01T22:27:24.160Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-01T22:28:12.176Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1580645, "meanTokensPerSec": 57.65, "genTokensPerSec": 57.65, "genTokens": 38126, "promptTokensPerSec": 760.87, "peakRssMb": 31203, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 356656, "output": 9099 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-craftbook-codemod-sweep", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 3 }, "trialId": "craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-11-01T22-28-39-209Z-uvix", "success": true, "includedInCapability": true, "durationMs": 6809460, "timing": { "startedAt": "2026-11-01T22:28:39.209Z", "finishedAt": "2026-11-02T00:22:08.669Z", "timeToFirstArtifactMs": 371531, "timeToLastArtifactWriteMs": 5802122, "timeToFirstTokenMs": 23026, "firstTurnTtftMs": 16477, "timeToFirstToolCallMs": 26533 }, "metrics": { "process": { "sampleCount": 1357, "peakRssMb": 18271, "peakCpuPercent": 136.8, "firstAtMs": 0, "lastAtMs": 6802741 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1357, "peakUsedMb": 83376, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 436259, "totalOutputTokens": 112340, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 48450, "todayTokensCached": 47899, "todayTokensOut": 545, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 297822, "totalTokensCached": 292615, "totalTokensOut": 16575, "totalCost": 0, "medianOutputTokensPerSec": 21.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 21.9, "turns": 12 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 48450, "cachedInputTokens": 47899, "outputTokens": 545, "cost": 0, "durationMs": 200367, "outputTokensPerSec": 17.602957708988665, "at": "2026-11-02T00:05:21.331Z" }, "lastUpdated": "2026-11-02T00:05:21.331Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T00:22:04.295Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6804022, "meanTokensPerSec": 20, "genTokensPerSec": 20, "genTokens": 112340, "promptTokensPerSec": null, "peakRssMb": 18271, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 297822, "output": 16575 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 437.5, "sampleCount": 24, "structuredSampleCount": 0, "firstTokenSampleCount": 24, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-developer-craftbook-refactor-module", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 3 }, "trialId": "craftbook-refactor-module-mlx-gemma4-12b-q4-2026-11-02T00-22-33-760Z-0qd0", "success": false, "includedInCapability": true, "durationMs": 1312928, "timing": { "startedAt": "2026-11-02T00:22:33.760Z", "finishedAt": "2026-11-02T00:44:26.688Z", "timeToFirstArtifactMs": 26880, "timeToLastArtifactWriteMs": 1259036, "timeToFirstTokenMs": 18977, "firstTurnTtftMs": 12441, "timeToFirstToolCallMs": 26878 }, "metrics": { "process": { "sampleCount": 261, "peakRssMb": 11713, "peakCpuPercent": 187.5, "firstAtMs": 0, "lastAtMs": 1306677 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 261, "peakUsedMb": 67072, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 161466, "totalOutputTokens": 11039, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 24, "todayTokensIn": 520845, "todayTokensCached": 429728, "todayTokensOut": 8310, "todayCost": 0, "totalTurns": 24, "totalTokensIn": 520845, "totalTokensCached": 429728, "totalTokensOut": 8310, "totalCost": 0, "medianOutputTokensPerSec": 45.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 45.6, "turns": 24 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 32986, "cachedInputTokens": 32133, "outputTokens": 174, "cost": 0, "durationMs": 4281, "outputTokensPerSec": 49.18381636120813, "at": "2026-11-02T00:43:32.796Z" }, "lastUpdated": "2026-11-02T00:43:32.796Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T00:44:24.812Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1309989, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 11039, "promptTokensPerSec": null, "peakRssMb": 11713, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 520845, "output": 8310 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1157, "sampleCount": 25, "structuredSampleCount": 0, "firstTokenSampleCount": 25, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-developer-craftbook-refactor-module", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 3 }, "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-11-02T00-44-50-301Z-da95", "success": false, "includedInCapability": true, "durationMs": 1655378, "timing": { "startedAt": "2026-11-02T00:44:50.302Z", "finishedAt": "2026-11-02T01:12:25.680Z", "timeToFirstArtifactMs": 17749, "timeToLastArtifactWriteMs": 1113547, "timeToFirstTokenMs": 14929, "firstTurnTtftMs": 8187, "timeToFirstToolCallMs": 17751 }, "metrics": { "process": { "sampleCount": 329, "peakRssMb": 31280, "peakCpuPercent": 121.4, "firstAtMs": 0, "lastAtMs": 1645754 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 329, "peakUsedMb": 77993, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 580392, "totalOutputTokens": 43953, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 104904, "todayTokensCached": 77795, "todayTokensOut": 4155, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 104904, "totalTokensCached": 77795, "totalTokensOut": 4155, "totalCost": 0, "medianOutputTokensPerSec": 89.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 89.7, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16200, "outputTokens": 102, "cost": 0, "durationMs": 33685, "outputTokensPerSec": 112.58901220420302, "at": "2026-11-02T00:49:29.200Z" }, "lastUpdated": "2026-11-02T00:49:29.200Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T01:12:17.776Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1646529, "meanTokensPerSec": 58.73, "genTokensPerSec": 58.73, "genTokens": 43953, "promptTokensPerSec": 813.97, "peakRssMb": 31280, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 104904, "output": 4155 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-craftbook-refactor-module", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 3 }, "trialId": "craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-11-02T01-12-47-279Z-i210", "success": false, "includedInCapability": true, "durationMs": 3028757, "timing": { "startedAt": "2026-11-02T01:12:47.280Z", "finishedAt": "2026-11-02T02:03:16.037Z", "timeToFirstArtifactMs": 41654, "timeToLastArtifactWriteMs": 3023576, "timeToFirstTokenMs": 22786, "firstTurnTtftMs": 16261, "timeToFirstToolCallMs": 41653 }, "metrics": { "process": { "sampleCount": 603, "peakRssMb": 18761, "peakCpuPercent": 120.6, "firstAtMs": 0, "lastAtMs": 3023210 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 603, "peakUsedMb": 84305, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 315539, "totalOutputTokens": 32522, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 24, "todayTokensIn": 442492, "todayTokensCached": 207563, "todayTokensOut": 21990, "todayCost": 0, "totalTurns": 24, "totalTokensIn": 442492, "totalTokensCached": 207563, "totalTokensOut": 21990, "totalCost": 0, "medianOutputTokensPerSec": 31.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.7, "turns": 24 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 27408, "cachedInputTokens": 2281, "outputTokens": 149, "cost": 0, "durationMs": 120998, "outputTokensPerSec": 18.00051350034438, "at": "2026-11-02T02:03:10.859Z" }, "lastUpdated": "2026-11-02T02:03:10.859Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T02:03:13.355Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3025008, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 32522, "promptTokensPerSec": null, "peakRssMb": 18761, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 442492, "output": 21990 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 500, "sampleCount": 24, "structuredSampleCount": 0, "firstTokenSampleCount": 24, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-craftbook-refactor-module", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 3 }, "trialId": "craftbook-refactor-module-qwen3-8-27b-q4-2026-11-02T02-04-02-214Z-osk7", "success": false, "includedInCapability": true, "durationMs": 2790357, "timing": { "startedAt": "2026-11-02T02:04:02.214Z", "finishedAt": "2026-11-02T02:50:32.571Z", "timeToFirstArtifactMs": 59261, "timeToLastArtifactWriteMs": 2693089, "timeToFirstTokenMs": 23942, "firstTurnTtftMs": 17062, "timeToFirstToolCallMs": 59260 }, "metrics": { "process": { "sampleCount": 555, "peakRssMb": 49012, "peakCpuPercent": 109.2, "firstAtMs": 0, "lastAtMs": 2779397 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 555, "peakUsedMb": 97569, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 301354, "totalOutputTokens": 41521, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 259539, "todayTokensCached": 169463, "todayTokensOut": 15898, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 259539, "totalTokensCached": 169463, "totalTokensOut": 15898, "totalCost": 0, "medianOutputTokensPerSec": 30.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.6, "turns": 13 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 46842, "outputTokens": 187, "cost": 0, "durationMs": 148818, "outputTokensPerSec": 32.25688970745082, "at": "2026-11-02T02:44:13.767Z" }, "lastUpdated": "2026-11-02T02:44:13.767Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T02:50:26.891Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2783448, "meanTokensPerSec": 25.28, "genTokensPerSec": 25.28, "genTokens": 41521, "promptTokensPerSec": 320.92, "peakRssMb": 49012, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 259539, "output": 15898 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-developer-craftbook-bug-fix-tdd", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 3 }, "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-11-02T02-50-54-966Z-8q0u", "success": false, "includedInCapability": true, "durationMs": 1525767, "timing": { "startedAt": "2026-11-02T02:50:54.966Z", "finishedAt": "2026-11-02T03:16:20.733Z", "timeToFirstArtifactMs": 33291, "timeToLastArtifactWriteMs": 1518461, "timeToFirstTokenMs": 19440, "firstTurnTtftMs": 12664, "timeToFirstToolCallMs": 24197 }, "metrics": { "process": { "sampleCount": 304, "peakRssMb": 28986, "peakCpuPercent": 108.5, "firstAtMs": 1, "lastAtMs": 1520225 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 304, "peakUsedMb": 76306, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 694368, "totalOutputTokens": 34591, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 24, "todayTokensIn": 467553, "todayTokensCached": 306042, "todayTokensOut": 8078, "todayCost": 0, "totalTurns": 24, "totalTokensIn": 467553, "totalTokensCached": 306042, "totalTokensOut": 8078, "totalCost": 0, "medianOutputTokensPerSec": 79.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 79.7, "turns": 24 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 25473, "outputTokens": 200, "cost": 0, "durationMs": 190248, "outputTokensPerSec": 81.43145865021015, "at": "2026-11-02T03:16:13.427Z" }, "lastUpdated": "2026-11-02T03:16:13.427Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T03:16:18.276Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1522262, "meanTokensPerSec": 63.17, "genTokensPerSec": 63.17, "genTokens": 34591, "promptTokensPerSec": 849.01, "peakRssMb": 28986, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 467553, "output": 8078 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-developer-craftbook-bug-fix-tdd", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 3 }, "trialId": "craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-11-02T03-16-42-447Z-3e1d", "success": false, "includedInCapability": true, "durationMs": 2356851, "timing": { "startedAt": "2026-11-02T03:16:42.448Z", "finishedAt": "2026-11-02T03:55:59.299Z", "timeToFirstArtifactMs": 54983, "timeToLastArtifactWriteMs": 2351072, "timeToFirstTokenMs": 29519, "firstTurnTtftMs": 23004, "timeToFirstToolCallMs": 37997 }, "metrics": { "process": { "sampleCount": 469, "peakRssMb": 18222, "peakCpuPercent": 112, "firstAtMs": 0, "lastAtMs": 2349977 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 469, "peakUsedMb": 83933, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 398079, "totalOutputTokens": 20171, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 416900, "todayTokensCached": 187772, "todayTokensOut": 11555, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 416900, "totalTokensCached": 187772, "totalTokensOut": 11555, "totalCost": 0, "medianOutputTokensPerSec": 29.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 29.2, "turns": 22 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 27646, "cachedInputTokens": 2253, "outputTokens": 453, "cost": 0, "durationMs": 143030, "outputTokensPerSec": 23.34128100964314, "at": "2026-11-02T03:55:53.522Z" }, "lastUpdated": "2026-11-02T03:55:53.522Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T03:55:56.789Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2353274, "meanTokensPerSec": 24, "genTokensPerSec": 24, "genTokens": 20171, "promptTokensPerSec": null, "peakRssMb": 18222, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 416900, "output": 11555 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 514, "sampleCount": 24, "structuredSampleCount": 0, "firstTokenSampleCount": 24, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-developer-craftbook-bug-fix-tdd", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 3 }, "trialId": "craftbook-bug-fix-tdd-qwen3-8-27b-q4-2026-11-02T03-56-21-204Z-crqq", "success": false, "includedInCapability": true, "durationMs": 3213586, "timing": { "startedAt": "2026-11-02T03:56:21.205Z", "finishedAt": "2026-11-02T04:49:54.791Z", "timeToFirstArtifactMs": 71144, "timeToLastArtifactWriteMs": 2965359, "timeToFirstTokenMs": 34435, "firstTurnTtftMs": 27712, "timeToFirstToolCallMs": 37846 }, "metrics": { "process": { "sampleCount": 640, "peakRssMb": 49814, "peakCpuPercent": 104.3, "firstAtMs": 0, "lastAtMs": 3205377 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 640, "peakUsedMb": 98319, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 189883, "totalOutputTokens": 54215, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 161529, "todayTokensCached": 118272, "todayTokensOut": 10632, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 161529, "totalTokensCached": 118272, "totalTokensOut": 10632, "totalCost": 0, "medianOutputTokensPerSec": 23.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 23.4, "turns": 11 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 24346, "cachedInputTokens": 24129, "outputTokens": 619, "cost": 0, "durationMs": 263718, "outputTokensPerSec": 23.968927891962654, "at": "2026-11-02T04:42:37.197Z" }, "lastUpdated": "2026-11-02T04:42:37.197Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T04:49:49.914Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3207659, "meanTokensPerSec": 22.45, "genTokensPerSec": 22.45, "genTokens": 54215, "promptTokensPerSec": 366.84, "peakRssMb": 49814, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 161529, "output": 10632 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-developer-craftbook-bug-fix-tdd", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 3 }, "trialId": "craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-11-02T04-50-37-295Z-n74w", "success": false, "includedInCapability": true, "durationMs": 1872155, "timing": { "startedAt": "2026-11-02T04:50:37.295Z", "finishedAt": "2026-11-02T05:21:49.450Z", "timeToFirstArtifactMs": 41008, "timeToLastArtifactWriteMs": 1605185, "timeToFirstTokenMs": 19846, "firstTurnTtftMs": 13214, "timeToFirstToolCallMs": 32419 }, "metrics": { "process": { "sampleCount": 373, "peakRssMb": 11912, "peakCpuPercent": 135, "firstAtMs": 0, "lastAtMs": 1867962 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 373, "peakUsedMb": 66750, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 511641, "totalOutputTokens": 23475, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 596542, "todayTokensCached": 311286, "todayTokensOut": 5156, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 596542, "totalTokensCached": 311286, "totalTokensOut": 5156, "totalCost": 0, "medianOutputTokensPerSec": 40.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 40.5, "turns": 22 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 646, "cachedInputTokens": 4, "outputTokens": 406, "cost": 0, "durationMs": 8283, "outputTokensPerSec": 54.53135595665429, "at": "2026-11-02T05:20:46.773Z" }, "lastUpdated": "2026-11-02T05:20:46.773Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T05:21:48.722Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1870272, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 23475, "promptTokensPerSec": null, "peakRssMb": 11912, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 596542, "output": 5156 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1017, "sampleCount": 28, "structuredSampleCount": 0, "firstTokenSampleCount": 28, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-find-vs-create", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 3 }, "trialId": "craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-11-02T05-22-30-018Z-aq07", "success": true, "includedInCapability": true, "durationMs": 56304, "timing": { "startedAt": "2026-11-02T05:22:30.018Z", "finishedAt": "2026-11-02T05:23:26.322Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 21483, "firstTurnTtftMs": 19788, "timeToFirstToolCallMs": 25184 }, "metrics": { "process": { "sampleCount": 11, "peakRssMb": 18032, "peakCpuPercent": 119.4, "firstAtMs": 0, "lastAtMs": 50173 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 11, "peakUsedMb": 84344, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 13084, "totalOutputTokens": 564, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T05:23:22.021Z" } }, "source": "engine-log" }, "derived": { "durationMs": 50946, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 564, "promptTokensPerSec": null, "peakRssMb": 18032, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 807, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-find-vs-create", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 3 }, "trialId": "craftbook-find-vs-create-qwen3-8-27b-q4-2026-11-02T05-23-49-225Z-kshg", "success": false, "includedInCapability": true, "durationMs": 1480608, "timing": { "startedAt": "2026-11-02T05:23:49.226Z", "finishedAt": "2026-11-02T05:48:29.834Z", "timeToFirstArtifactMs": 633996, "timeToLastArtifactWriteMs": 1474275, "timeToFirstTokenMs": 28419, "firstTurnTtftMs": 26525, "timeToFirstToolCallMs": 35838 }, "metrics": { "process": { "sampleCount": 294, "peakRssMb": 49848, "peakCpuPercent": 123.3, "firstAtMs": 0, "lastAtMs": 1469747 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 294, "peakUsedMb": 105631, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 224632, "totalOutputTokens": 21747, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 137084, "todayTokensCached": 71266, "todayTokensOut": 4743, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 137084, "totalTokensCached": 71266, "totalTokensOut": 4743, "totalCost": 0, "medianOutputTokensPerSec": 26.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.1, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 30781, "outputTokens": 678, "cost": 0, "durationMs": 104679, "outputTokensPerSec": 32.05867504668752, "at": "2026-11-02T05:48:23.501Z" }, "lastUpdated": "2026-11-02T05:48:23.501Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T05:48:24.324Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1474054, "meanTokensPerSec": 27.47, "genTokensPerSec": 27.47, "genTokens": 21747, "promptTokensPerSec": 359.75, "peakRssMb": 49848, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 137084, "output": 4743 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-find-vs-create", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 3 }, "trialId": "craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-11-02T05-48-51-395Z-5dc5", "success": true, "includedInCapability": true, "durationMs": 338973, "timing": { "startedAt": "2026-11-02T05:48:51.395Z", "finishedAt": "2026-11-02T05:54:30.368Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 17115, "firstTurnTtftMs": 15376, "timeToFirstToolCallMs": 56965 }, "metrics": { "process": { "sampleCount": 67, "peakRssMb": 10673, "peakCpuPercent": 110.5, "firstAtMs": 0, "lastAtMs": 331219 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 67, "peakUsedMb": 68209, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 11042, "totalOutputTokens": 11058, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 21303, "todayTokensCached": 20954, "todayTokensOut": 8251, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 21303, "totalTokensCached": 20954, "totalTokensOut": 8251, "totalCost": 0, "medianOutputTokensPerSec": 38.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 38.7, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10978, "cachedInputTokens": 10865, "outputTokens": 59, "cost": 0, "durationMs": 53874, "outputTokensPerSec": 40.04229640098977, "at": "2026-11-02T05:54:21.644Z" }, "lastUpdated": "2026-11-02T05:54:21.644Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T05:54:25.211Z" } }, "source": "engine-log" }, "derived": { "durationMs": 332747, "meanTokensPerSec": 38, "genTokensPerSec": 38, "genTokens": 11058, "promptTokensPerSec": null, "peakRssMb": 10673, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 21303, "output": 8251 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 694.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-find-vs-create", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 3 }, "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-11-02T05-55-06-577Z-shdf", "success": true, "includedInCapability": true, "durationMs": 84713, "timing": { "startedAt": "2026-11-02T05:55:06.577Z", "finishedAt": "2026-11-02T05:56:31.290Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 13304, "firstTurnTtftMs": 11454, "timeToFirstToolCallMs": 23596 }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 17526, "peakCpuPercent": 116.9, "firstAtMs": 0, "lastAtMs": 70231 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 65019, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 12315, "totalOutputTokens": 3775, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 22874, "todayTokensCached": 22457, "todayTokensOut": 2175, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 22874, "totalTokensCached": 22457, "totalTokensOut": 2175, "totalCost": 0, "medianOutputTokensPerSec": 74.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 74.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12294, "cachedInputTokens": 12122, "outputTokens": 44, "cost": 0, "durationMs": 17795, "outputTokensPerSec": 66.51188474573127, "at": "2026-11-02T05:56:15.775Z" }, "lastUpdated": "2026-11-02T05:56:15.775Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T05:56:18.623Z" } }, "source": "engine-log" }, "derived": { "durationMs": 71004, "meanTokensPerSec": 72.47, "genTokensPerSec": 72.47, "genTokens": 3775, "promptTokensPerSec": 1123.01, "peakRssMb": 17526, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22874, "output": 2175 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-edit-midtask", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 3 }, "trialId": "craftbook-edit-midtask-qwen3-8-27b-q4-2026-11-02T05-56-56-577Z-y4i2", "success": true, "includedInCapability": true, "durationMs": 478569, "timing": { "startedAt": "2026-11-02T05:56:56.577Z", "finishedAt": "2026-11-02T06:04:55.146Z", "timeToFirstArtifactMs": 317544, "timeToLastArtifactWriteMs": 317547, "timeToFirstTokenMs": 45980, "firstTurnTtftMs": 44121, "timeToFirstToolCallMs": 46974 }, "metrics": { "process": { "sampleCount": 96, "peakRssMb": 47117, "peakCpuPercent": 116.4, "firstAtMs": 0, "lastAtMs": 476544 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 96, "peakUsedMb": 101717, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 66450, "totalOutputTokens": 8001, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 82219, "todayTokensCached": 63875, "todayTokensOut": 1668, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 82219, "totalTokensCached": 63875, "totalTokensOut": 1668, "totalCost": 0, "medianOutputTokensPerSec": 28.1, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 28.1, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7304, "outputTokens": 736, "cost": 0, "durationMs": 42660, "outputTokensPerSec": 29.870061574591965, "at": "2026-11-02T06:04:51.227Z" }, "lastUpdated": "2026-11-02T06:04:51.227Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T06:04:54.722Z" } }, "source": "engine-log" }, "derived": { "durationMs": 477092, "meanTokensPerSec": 28.45, "genTokensPerSec": 28.45, "genTokens": 8001, "promptTokensPerSec": 395.23, "peakRssMb": 47117, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 82219, "output": 1668 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-edit-midtask", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 3 }, "trialId": "craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-11-02T06-05-17-170Z-axk4", "success": false, "includedInCapability": true, "durationMs": 898093, "timing": { "startedAt": "2026-11-02T06:05:17.171Z", "finishedAt": "2026-11-02T06:20:15.264Z", "timeToFirstArtifactMs": 195024, "timeToLastArtifactWriteMs": 891854, "timeToFirstTokenMs": 20589, "firstTurnTtftMs": 18910, "timeToFirstToolCallMs": 33800 }, "metrics": { "process": { "sampleCount": 179, "peakRssMb": 11546, "peakCpuPercent": 118.2, "firstAtMs": 0, "lastAtMs": 893302 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 179, "peakUsedMb": 68214, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 91213, "totalOutputTokens": 23979, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 285640, "todayTokensCached": 255264, "todayTokensOut": 13609, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 285640, "totalTokensCached": 255264, "totalTokensOut": 13609, "totalCost": 0, "medianOutputTokensPerSec": 37.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 37.7, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8378, "cachedInputTokens": 1495, "outputTokens": 77, "cost": 0, "durationMs": 22533, "outputTokensPerSec": 45.84579998129103, "at": "2026-11-02T06:20:09.029Z" }, "lastUpdated": "2026-11-02T06:20:09.029Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T06:20:12.065Z" } }, "source": "engine-log" }, "derived": { "durationMs": 893828, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 23979, "promptTokensPerSec": null, "peakRssMb": 11546, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 285640, "output": 13609 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 547.5, "sampleCount": 18, "structuredSampleCount": 0, "firstTokenSampleCount": 18, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-edit-midtask", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 3 }, "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-11-02T06-20-40-166Z-kx0m", "success": true, "includedInCapability": true, "durationMs": 649446, "timing": { "startedAt": "2026-11-02T06:20:40.167Z", "finishedAt": "2026-11-02T06:31:29.613Z", "timeToFirstArtifactMs": 49822, "timeToLastArtifactWriteMs": 646977, "timeToFirstTokenMs": 18929, "firstTurnTtftMs": 17020, "timeToFirstToolCallMs": 21116 }, "metrics": { "process": { "sampleCount": 130, "peakRssMb": 28455, "peakCpuPercent": 114.1, "firstAtMs": 0, "lastAtMs": 647110 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 130, "peakUsedMb": 84711, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 183003, "totalOutputTokens": 23229, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 102949, "todayTokensCached": 69256, "todayTokensOut": 3144, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 102949, "totalTokensCached": 69256, "totalTokensOut": 3144, "totalCost": 0, "medianOutputTokensPerSec": 52.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 52.9, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8250, "cachedInputTokens": 8121, "outputTokens": 30, "cost": 0, "durationMs": 185249, "outputTokensPerSec": 84.66584725697253, "at": "2026-11-02T06:31:27.143Z" }, "lastUpdated": "2026-11-02T06:31:27.143Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T06:31:28.868Z" } }, "source": "engine-log" }, "derived": { "durationMs": 647651, "meanTokensPerSec": 60.97, "genTokensPerSec": 60.97, "genTokens": 23229, "promptTokensPerSec": 841.48, "peakRssMb": 28455, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 102949, "output": 3144 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-edit-midtask", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 3 }, "trialId": "craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-11-02T06-31-52-526Z-b0il", "success": true, "includedInCapability": true, "durationMs": 952222, "timing": { "startedAt": "2026-11-02T06:31:52.526Z", "finishedAt": "2026-11-02T06:47:44.748Z", "timeToFirstArtifactMs": 871541, "timeToLastArtifactWriteMs": 925287, "timeToFirstTokenMs": 43082, "firstTurnTtftMs": 41381, "timeToFirstToolCallMs": 44490 }, "metrics": { "process": { "sampleCount": 190, "peakRssMb": 18148, "peakCpuPercent": 119.7, "firstAtMs": 0, "lastAtMs": 948099 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 190, "peakUsedMb": 85542, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 36930, "totalOutputTokens": 18693, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 132508, "todayTokensCached": 125724, "todayTokensOut": 4499, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 132508, "totalTokensCached": 125724, "totalTokensOut": 4499, "totalCost": 0, "medianOutputTokensPerSec": 24.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 24.7, "turns": 7 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 26733, "cachedInputTokens": 26458, "outputTokens": 100, "cost": 0, "durationMs": 10520, "outputTokensPerSec": 24.669913232590538, "at": "2026-11-02T06:47:37.712Z" }, "lastUpdated": "2026-11-02T06:47:37.712Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T06:47:42.236Z" } }, "source": "engine-log" }, "derived": { "durationMs": 948643, "meanTokensPerSec": 23, "genTokensPerSec": 23, "genTokens": 18693, "promptTokensPerSec": null, "peakRssMb": 18148, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 132508, "output": 4499 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 291, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-invoice-run", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 3 }, "trialId": "craftbook-invoice-run-mlx-gemma4-12b-q4-2026-11-02T06-48-06-622Z-syhp", "success": true, "includedInCapability": true, "durationMs": 1181155, "timing": { "startedAt": "2026-11-02T06:48:06.623Z", "finishedAt": "2026-11-02T07:07:47.778Z", "timeToFirstArtifactMs": 41803, "timeToLastArtifactWriteMs": 1164797, "timeToFirstTokenMs": 20135, "firstTurnTtftMs": 13626, "timeToFirstToolCallMs": 25967 }, "metrics": { "process": { "sampleCount": 233, "peakRssMb": 14176, "peakCpuPercent": 134.5, "firstAtMs": 0, "lastAtMs": 1164377 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 233, "peakUsedMb": 70400, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 357408, "totalOutputTokens": 27008, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 23, "todayTokensIn": 383096, "todayTokensCached": 262280, "todayTokensOut": 8227, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 383096, "totalTokensCached": 262280, "totalTokensOut": 8227, "totalCost": 0, "medianOutputTokensPerSec": 36.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.7, "turns": 23 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15534, "cachedInputTokens": 9705, "outputTokens": 105, "cost": 0, "durationMs": 22850, "outputTokensPerSec": 49.854382662059635, "at": "2026-11-02T07:07:31.420Z" }, "lastUpdated": "2026-11-02T07:07:31.420Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T07:07:33.871Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1166184, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 27008, "promptTokensPerSec": null, "peakRssMb": 14176, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 383096, "output": 8227 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 1009, "sampleCount": 23, "structuredSampleCount": 0, "firstTokenSampleCount": 23, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-invoice-run", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 3 }, "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-11-02T07-08-09-564Z-8qfa", "success": true, "includedInCapability": true, "durationMs": 238489, "timing": { "startedAt": "2026-11-02T07:08:09.565Z", "finishedAt": "2026-11-02T07:12:08.054Z", "timeToFirstArtifactMs": 22753, "timeToLastArtifactWriteMs": 227634, "timeToFirstTokenMs": 15918, "firstTurnTtftMs": 9194, "timeToFirstToolCallMs": 18357 }, "metrics": { "process": { "sampleCount": 47, "peakRssMb": 27947, "peakCpuPercent": 147.5, "firstAtMs": 0, "lastAtMs": 230755 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 47, "peakUsedMb": 82504, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 89018, "totalOutputTokens": 9278, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 74001, "todayTokensCached": 61881, "todayTokensOut": 478, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 74001, "totalTokensCached": 61881, "totalTokensOut": 478, "totalCost": 0, "medianOutputTokensPerSec": 100.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100.9, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16803, "cachedInputTokens": 9741, "outputTokens": 79, "cost": 0, "durationMs": 51051, "outputTokensPerSec": 67.83204437954987, "at": "2026-11-02T07:11:57.199Z" }, "lastUpdated": "2026-11-02T07:11:57.199Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T07:12:01.790Z" } }, "source": "engine-log" }, "derived": { "durationMs": 231280, "meanTokensPerSec": 100.31, "genTokensPerSec": 100.31, "genTokens": 9278, "promptTokensPerSec": 1159.42, "peakRssMb": 27947, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 74001, "output": 478 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-invoice-run", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 3 }, "trialId": "craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-11-02T07-12-29-556Z-ei5a", "success": true, "includedInCapability": true, "durationMs": 1087919, "timing": { "startedAt": "2026-11-02T07:12:29.556Z", "finishedAt": "2026-11-02T07:30:37.475Z", "timeToFirstArtifactMs": 79068, "timeToLastArtifactWriteMs": 1066556, "timeToFirstTokenMs": 26724, "firstTurnTtftMs": 20219, "timeToFirstToolCallMs": 28384 }, "metrics": { "process": { "sampleCount": 214, "peakRssMb": 19359, "peakCpuPercent": 118.7, "firstAtMs": 0, "lastAtMs": 1068552 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 214, "peakUsedMb": 83851, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 92382, "totalOutputTokens": 21126, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 91694, "todayTokensCached": 88781, "todayTokensOut": 1267, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 91694, "totalTokensCached": 88781, "totalTokensOut": 1267, "totalCost": 0, "medianOutputTokensPerSec": 30.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 30.6, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 23229, "cachedInputTokens": 20810, "outputTokens": 686, "cost": 0, "durationMs": 459840, "outputTokensPerSec": 28.252026498858328, "at": "2026-11-02T07:30:16.112Z" }, "lastUpdated": "2026-11-02T07:30:16.112Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T07:30:19.863Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1069346, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 21126, "promptTokensPerSec": null, "peakRssMb": 19359, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 91694, "output": 1267 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 555.5, "sampleCount": 6, "structuredSampleCount": 0, "firstTokenSampleCount": 6, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-invoice-run", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 3 }, "trialId": "craftbook-invoice-run-qwen3-8-27b-q4-2026-11-02T07-31-00-191Z-flbn", "success": true, "includedInCapability": true, "durationMs": 542994, "timing": { "startedAt": "2026-11-02T07:31:00.192Z", "finishedAt": "2026-11-02T07:40:03.186Z", "timeToFirstArtifactMs": 81346, "timeToLastArtifactWriteMs": 529556, "timeToFirstTokenMs": 31574, "firstTurnTtftMs": 24874, "timeToFirstToolCallMs": 33160 }, "metrics": { "process": { "sampleCount": 107, "peakRssMb": 48534, "peakCpuPercent": 118.2, "firstAtMs": 0, "lastAtMs": 531735 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 107, "peakUsedMb": 104720, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 81487, "totalOutputTokens": 10380, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 82492, "todayTokensCached": 80451, "todayTokensOut": 1004, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 82492, "totalTokensCached": 80451, "totalTokensOut": 1004, "totalCost": 0, "medianOutputTokensPerSec": 39.8, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 39.8, "turns": 6 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 21485, "cachedInputTokens": 20222, "outputTokens": 464, "cost": 0, "durationMs": 112839, "outputTokensPerSec": 33.82505811444515, "at": "2026-11-02T07:39:49.748Z" }, "lastUpdated": "2026-11-02T07:39:49.748Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T07:39:53.755Z" } }, "source": "engine-log" }, "derived": { "durationMs": 532512, "meanTokensPerSec": 37.66, "genTokensPerSec": 37.66, "genTokens": 10380, "promptTokensPerSec": 404, "peakRssMb": 48534, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 82492, "output": 1004 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-author-linear", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 3 }, "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-11-02T07-40-24-910Z-yxge", "success": false, "includedInCapability": true, "durationMs": 1174539, "timing": { "startedAt": "2026-11-02T07:40:24.910Z", "finishedAt": "2026-11-02T07:59:59.449Z", "timeToFirstArtifactMs": 425451, "timeToLastArtifactWriteMs": 1169356, "timeToFirstTokenMs": 18468, "firstTurnTtftMs": 16739, "timeToFirstToolCallMs": 23695 }, "metrics": { "process": { "sampleCount": 234, "peakRssMb": 29754, "peakCpuPercent": 135.6, "firstAtMs": 0, "lastAtMs": 1168800 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 234, "peakUsedMb": 85502, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 298608, "totalOutputTokens": 40816, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 27, "todayTokensIn": 609320, "todayTokensCached": 489596, "todayTokensOut": 8175, "todayCost": 0, "totalTurns": 27, "totalTokensIn": 609320, "totalTokensCached": 489596, "totalTokensOut": 8175, "totalCost": 0, "medianOutputTokensPerSec": 51.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 51.9, "turns": 27 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11333, "outputTokens": 337, "cost": 0, "durationMs": 56516, "outputTokensPerSec": 83.19061768290857, "at": "2026-11-02T07:59:54.266Z" }, "lastUpdated": "2026-11-02T07:59:54.266Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T07:59:58.451Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1172598, "meanTokensPerSec": 56.5, "genTokensPerSec": 56.5, "genTokens": 40816, "promptTokensPerSec": 871.25, "peakRssMb": 29754, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 609320, "output": 8175 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-author-linear", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 3 }, "trialId": "craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-11-02T08-00-21-017Z-wtzc", "success": false, "includedInCapability": true, "durationMs": 5573211, "timing": { "startedAt": "2026-11-02T08:00:21.018Z", "finishedAt": "2026-11-02T09:33:14.229Z", "timeToFirstArtifactMs": 2450965, "timeToLastArtifactWriteMs": 4957809, "timeToFirstTokenMs": 41372, "firstTurnTtftMs": 39668, "timeToFirstToolCallMs": 58838 }, "metrics": { "process": { "sampleCount": 1108, "peakRssMb": 19863, "peakCpuPercent": 173.5, "firstAtMs": 0, "lastAtMs": 5557628 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1108, "peakUsedMb": 91724, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 210350, "totalOutputTokens": 73753, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 467617, "todayTokensCached": 435231, "todayTokensOut": 11233, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 467617, "totalTokensCached": 435231, "totalTokensOut": 11233, "totalCost": 0, "medianOutputTokensPerSec": 17.9, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 17.9, "turns": 13 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9167, "cachedInputTokens": 9105, "outputTokens": 83, "cost": 0, "durationMs": 183030, "outputTokensPerSec": 30.338504166144766, "at": "2026-11-02T09:17:34.748Z" }, "lastUpdated": "2026-11-02T09:17:34.748Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T09:33:02.009Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5559924, "meanTokensPerSec": 9.5, "genTokensPerSec": 9.5, "genTokens": 73753, "promptTokensPerSec": null, "peakRssMb": 19863, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 467617, "output": 11233 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 503, "sampleCount": 17, "structuredSampleCount": 0, "firstTokenSampleCount": 17, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-author-linear", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 3 }, "trialId": "craftbook-author-linear-qwen3-8-27b-q4-2026-11-02T09-33-40-109Z-rdkp", "success": false, "includedInCapability": true, "durationMs": 2355825, "timing": { "startedAt": "2026-11-02T09:33:40.110Z", "finishedAt": "2026-11-02T10:12:55.935Z", "timeToFirstArtifactMs": 812221, "timeToLastArtifactWriteMs": 2350163, "timeToFirstTokenMs": 47076, "firstTurnTtftMs": 45127, "timeToFirstToolCallMs": 52225 }, "metrics": { "process": { "sampleCount": 470, "peakRssMb": 50297, "peakCpuPercent": 112.4, "firstAtMs": 0, "lastAtMs": 2352595 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 470, "peakUsedMb": 107046, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 372589, "totalOutputTokens": 31719, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 225419, "todayTokensCached": 159211, "todayTokensOut": 5291, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 225419, "totalTokensCached": 159211, "totalTokensOut": 5291, "totalCost": 0, "medianOutputTokensPerSec": 27, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 27, "turns": 9 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 5001, "outputTokens": 1126, "cost": 0, "durationMs": 491552, "outputTokensPerSec": 42.26877281128255, "at": "2026-11-02T10:12:50.273Z" }, "lastUpdated": "2026-11-02T10:12:50.273Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T10:12:54.879Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2353621, "meanTokensPerSec": 25.87, "genTokensPerSec": 25.87, "genTokens": 31719, "promptTokensPerSec": 355.14, "peakRssMb": 50297, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 225419, "output": 5291 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-author-linear", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 3 }, "trialId": "craftbook-author-linear-mlx-gemma4-12b-q4-2026-11-02T10-13-18-192Z-yodh", "success": false, "includedInCapability": true, "durationMs": 1163177, "timing": { "startedAt": "2026-11-02T10:13:18.193Z", "finishedAt": "2026-11-02T10:32:41.370Z", "timeToFirstArtifactMs": 734825, "timeToLastArtifactWriteMs": 1157955, "timeToFirstTokenMs": 20252, "firstTurnTtftMs": 18576, "timeToFirstToolCallMs": 32999 }, "metrics": { "process": { "sampleCount": 231, "peakRssMb": 11292, "peakCpuPercent": 114.3, "firstAtMs": 0, "lastAtMs": 1155902 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 231, "peakUsedMb": 68692, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 30874, "totalOutputTokens": 21048, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 150439, "todayTokensCached": 146367, "todayTokensOut": 8816, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 150439, "totalTokensCached": 146367, "totalTokensOut": 8816, "totalCost": 0, "medianOutputTokensPerSec": 42.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 42.7, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 31337, "cachedInputTokens": 31031, "outputTokens": 8192, "cost": 0, "durationMs": 309578, "outputTokensPerSec": 33.15417656245013, "at": "2026-11-02T10:32:36.148Z" }, "lastUpdated": "2026-11-02T10:32:36.148Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T10:32:39.971Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1160710, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 21048, "promptTokensPerSec": null, "peakRssMb": 11292, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 150439, "output": 8816 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 903, "sampleCount": 7, "structuredSampleCount": 0, "firstTokenSampleCount": 7, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-route-multi", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 3 }, "trialId": "craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-11-02T10-33-04-013Z-5lji", "success": false, "includedInCapability": true, "durationMs": 739196, "timing": { "startedAt": "2026-11-02T10:33:04.013Z", "finishedAt": "2026-11-02T10:45:23.209Z", "timeToFirstArtifactMs": 524175, "timeToLastArtifactWriteMs": 734017, "timeToFirstTokenMs": 29035, "firstTurnTtftMs": 27306, "timeToFirstToolCallMs": 43859 }, "metrics": { "process": { "sampleCount": 147, "peakRssMb": 18747, "peakCpuPercent": 129, "firstAtMs": 0, "lastAtMs": 732456 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 147, "peakUsedMb": 84238, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 80629, "totalOutputTokens": 14265, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 57353, "todayTokensCached": 26859, "todayTokensOut": 6696, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 57353, "totalTokensCached": 26859, "totalTokensOut": 6696, "totalCost": 0, "medianOutputTokensPerSec": 36.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 36.7, "turns": 8 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 8787, "cachedInputTokens": 1214, "outputTokens": 347, "cost": 0, "durationMs": 23312, "outputTokensPerSec": 36.332429389223385, "at": "2026-11-02T10:45:18.029Z" }, "lastUpdated": "2026-11-02T10:45:18.029Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T10:45:20.329Z" } }, "source": "engine-log" }, "derived": { "durationMs": 735248, "meanTokensPerSec": 28, "genTokensPerSec": 28, "genTokens": 14265, "promptTokensPerSec": null, "peakRssMb": 18747, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 57353, "output": 6696 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 554, "sampleCount": 9, "structuredSampleCount": 0, "firstTokenSampleCount": 9, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-route-multi", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 3 }, "trialId": "craftbook-route-multi-qwen3-8-27b-q4-2026-11-02T10-45-47-017Z-yliw", "success": true, "includedInCapability": true, "durationMs": 1473181, "timing": { "startedAt": "2026-11-02T10:45:47.018Z", "finishedAt": "2026-11-02T11:10:20.199Z", "timeToFirstArtifactMs": 239668, "timeToLastArtifactWriteMs": 1363909, "timeToFirstTokenMs": 34251, "firstTurnTtftMs": 32351, "timeToFirstToolCallMs": 43035 }, "metrics": { "process": { "sampleCount": 293, "peakRssMb": 50721, "peakCpuPercent": 112.1, "firstAtMs": 0, "lastAtMs": 1465350 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 293, "peakUsedMb": 105310, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 195639, "totalOutputTokens": 21503, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 200877, "todayTokensCached": 178844, "todayTokensOut": 4572, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 200877, "totalTokensCached": 178844, "totalTokensOut": 4572, "totalCost": 0, "medianOutputTokensPerSec": 32.2, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 32.2, "turns": 14 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 7738, "cachedInputTokens": 7387, "outputTokens": 86, "cost": 0, "durationMs": 37887, "outputTokensPerSec": 33.03024521265455, "at": "2026-11-02T11:10:11.963Z" }, "lastUpdated": "2026-11-02T11:10:11.963Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T11:10:17.237Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1469169, "meanTokensPerSec": 28.77, "genTokensPerSec": 28.77, "genTokens": 21503, "promptTokensPerSec": 416.68, "peakRssMb": 50721, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 200877, "output": 4572 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-route-multi", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 3 }, "trialId": "craftbook-route-multi-mlx-gemma4-12b-q4-2026-11-02T11-10-42-097Z-fgan", "success": false, "includedInCapability": true, "durationMs": 1816310, "timing": { "startedAt": "2026-11-02T11:10:42.098Z", "finishedAt": "2026-11-02T11:40:58.408Z", "timeToFirstArtifactMs": 713245, "timeToLastArtifactWriteMs": 1634825, "timeToFirstTokenMs": 14723, "firstTurnTtftMs": 12998, "timeToFirstToolCallMs": 41299 }, "metrics": { "process": { "sampleCount": 362, "peakRssMb": 14252, "peakCpuPercent": 117.2, "firstAtMs": 0, "lastAtMs": 1813099 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 362, "peakUsedMb": 68776, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 104400, "totalOutputTokens": 41588, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 22, "todayTokensIn": 229069, "todayTokensCached": 219115, "todayTokensOut": 31456, "todayCost": 0, "totalTurns": 22, "totalTokensIn": 229069, "totalTokensCached": 219115, "totalTokensOut": 31456, "totalCost": 0, "medianOutputTokensPerSec": 39.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 39.5, "turns": 22 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12809, "cachedInputTokens": 12610, "outputTokens": 39, "cost": 0, "durationMs": 21010, "outputTokensPerSec": 45.36703124710773, "at": "2026-11-02T11:38:20.230Z" }, "lastUpdated": "2026-11-02T11:38:20.230Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T11:40:57.562Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1814401, "meanTokensPerSec": 41, "genTokensPerSec": 41, "genTokens": 41588, "promptTokensPerSec": null, "peakRssMb": 14252, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 229069, "output": 31456 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 712, "sampleCount": 22, "structuredSampleCount": 0, "firstTokenSampleCount": 22, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-route-multi", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 3 }, "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-11-02T11-41-21-140Z-hd40", "success": true, "includedInCapability": true, "durationMs": 955204, "timing": { "startedAt": "2026-11-02T11:41:21.141Z", "finishedAt": "2026-11-02T11:57:16.345Z", "timeToFirstArtifactMs": 113605, "timeToLastArtifactWriteMs": 871164, "timeToFirstTokenMs": 12087, "firstTurnTtftMs": 10278, "timeToFirstToolCallMs": 16820 }, "metrics": { "process": { "sampleCount": 189, "peakRssMb": 28689, "peakCpuPercent": 114.6, "firstAtMs": 0, "lastAtMs": 943499 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 189, "peakUsedMb": 85205, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 216248, "totalOutputTokens": 33043, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 26, "todayTokensIn": 252819, "todayTokensCached": 174376, "todayTokensOut": 12872, "todayCost": 0, "totalTurns": 26, "totalTokensIn": 252819, "totalTokensCached": 174376, "totalTokensOut": 12872, "totalCost": 0, "medianOutputTokensPerSec": 70.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.5, "turns": 26 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8208, "cachedInputTokens": 7960, "outputTokens": 33, "cost": 0, "durationMs": 13761, "outputTokensPerSec": 122.91949157425296, "at": "2026-11-02T11:57:06.719Z" }, "lastUpdated": "2026-11-02T11:57:06.719Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T11:57:08.880Z" } }, "source": "engine-log" }, "derived": { "durationMs": 946799, "meanTokensPerSec": 65.37, "genTokensPerSec": 65.37, "genTokens": 33043, "promptTokensPerSec": 1022.66, "peakRssMb": 28689, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 252819, "output": 12872 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-export-generalize", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 3 }, "trialId": "craftbook-export-generalize-qwen3-8-27b-q4-2026-11-02T11-57-38-352Z-szcf", "success": true, "includedInCapability": true, "durationMs": 2000002, "timing": { "startedAt": "2026-11-02T11:57:38.352Z", "finishedAt": "2026-11-02T12:30:58.354Z", "timeToFirstArtifactMs": 623831, "timeToLastArtifactWriteMs": 1793295, "timeToFirstTokenMs": 40233, "firstTurnTtftMs": 38462, "timeToFirstToolCallMs": 95243 }, "metrics": { "process": { "sampleCount": 399, "peakRssMb": 49647, "peakCpuPercent": 152.9, "firstAtMs": 0, "lastAtMs": 1996544 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 399, "peakUsedMb": 105538, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 241967, "totalOutputTokens": 33186, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 239295, "todayTokensCached": 165476, "todayTokensOut": 6034, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 239295, "totalTokensCached": 165476, "totalTokensOut": 6034, "totalCost": 0, "medianOutputTokensPerSec": 26.5, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.5, "turns": 11 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 27477, "cachedInputTokens": 27411, "outputTokens": 176, "cost": 0, "durationMs": 36945, "outputTokensPerSec": 26.22554598964196, "at": "2026-11-02T12:28:11.830Z" }, "lastUpdated": "2026-11-02T12:28:11.830Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T12:30:56.468Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1997066, "meanTokensPerSec": 25.52, "genTokensPerSec": 25.52, "genTokens": 33186, "promptTokensPerSec": 384.31, "peakRssMb": 49647, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 239295, "output": 6034 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-export-generalize", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 3 }, "trialId": "craftbook-export-generalize-mlx-gemma4-12b-q4-2026-11-02T12-31-24-072Z-dtnw", "success": false, "includedInCapability": true, "durationMs": 128841, "timing": { "startedAt": "2026-11-02T12:31:24.073Z", "finishedAt": "2026-11-02T12:33:32.914Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 21086, "firstTurnTtftMs": 19207, "timeToFirstToolCallMs": 122385 }, "metrics": { "process": { "sampleCount": 26, "peakRssMb": 10402, "peakCpuPercent": 110.3, "firstAtMs": 0, "lastAtMs": 125535 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 69398, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 904, "totalOutputTokens": 3530, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T12:33:31.922Z" } }, "source": "engine-log" }, "derived": { "durationMs": 126583, "meanTokensPerSec": 39, "genTokensPerSec": 39, "genTokens": 3530, "promptTokensPerSec": null, "peakRssMb": 10402, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 999, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-export-generalize", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 3 }, "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-11-02T12-33-58-762Z-3fi4", "success": false, "includedInCapability": true, "durationMs": 818479, "timing": { "startedAt": "2026-11-02T12:33:58.763Z", "finishedAt": "2026-11-02T12:47:37.242Z", "timeToFirstArtifactMs": 66846, "timeToLastArtifactWriteMs": 511901, "timeToFirstTokenMs": 18344, "firstTurnTtftMs": 16459, "timeToFirstToolCallMs": 23607 }, "metrics": { "process": { "sampleCount": 163, "peakRssMb": 23115, "peakCpuPercent": 98.5, "firstAtMs": 0, "lastAtMs": 814499 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 163, "peakUsedMb": 78238, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 23942, "totalOutputTokens": 12225, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 103224, "todayTokensCached": 101409, "todayTokensOut": 805, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 103224, "totalTokensCached": 101409, "totalTokensOut": 805, "totalCost": 0, "medianOutputTokensPerSec": 63.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 63.5, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 23844, "cachedInputTokens": 23415, "outputTokens": 147, "cost": 0, "durationMs": 26869, "outputTokensPerSec": 63.517116775349095, "at": "2026-11-02T12:42:30.664Z" }, "lastUpdated": "2026-11-02T12:42:30.664Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T12:47:37.124Z" } }, "source": "engine-log" }, "derived": { "durationMs": 817312, "meanTokensPerSec": 74.84, "genTokensPerSec": 74.84, "genTokens": 12225, "promptTokensPerSec": 874.74, "peakRssMb": 23115, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 103224, "output": 805 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-export-generalize", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 3 }, "trialId": "craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-11-02T12-47-58-690Z-f716", "success": true, "includedInCapability": true, "durationMs": 6695595, "timing": { "startedAt": "2026-11-02T12:47:58.691Z", "finishedAt": "2026-11-02T14:39:34.286Z", "timeToFirstArtifactMs": 1018100, "timeToLastArtifactWriteMs": 6683176, "timeToFirstTokenMs": 31411, "firstTurnTtftMs": 29873, "timeToFirstToolCallMs": 121427 }, "metrics": { "process": { "sampleCount": 1332, "peakRssMb": 18894, "peakCpuPercent": 170.3, "firstAtMs": 0, "lastAtMs": 6687315 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 1332, "peakUsedMb": 85934, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 153040, "totalOutputTokens": 85073, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 329063, "todayTokensCached": 288097, "todayTokensOut": 19815, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 329063, "totalTokensCached": 288097, "totalTokensOut": 19815, "totalCost": 0, "medianOutputTokensPerSec": 26.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 26.6, "turns": 16 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 14905, "cachedInputTokens": 14822, "outputTokens": 61, "cost": 0, "durationMs": 1122122, "outputTokensPerSec": 26.320970929501918, "at": "2026-11-02T14:39:24.684Z" }, "lastUpdated": "2026-11-02T14:39:24.684Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T14:39:28.753Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6689105, "meanTokensPerSec": 12, "genTokensPerSec": 12, "genTokens": 85073, "promptTokensPerSec": null, "peakRssMb": 18894, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 329063, "output": 19815 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 483, "sampleCount": 18, "structuredSampleCount": 0, "firstTokenSampleCount": 18, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-author-params", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 3 }, "trialId": "craftbook-author-params-mlx-gemma4-12b-q4-2026-11-02T14-39-55-839Z-slk9", "success": false, "includedInCapability": true, "durationMs": 996073, "timing": { "startedAt": "2026-11-02T14:39:55.839Z", "finishedAt": "2026-11-02T14:56:31.912Z", "timeToFirstArtifactMs": 87490, "timeToLastArtifactWriteMs": 490812, "timeToFirstTokenMs": 20231, "firstTurnTtftMs": 18664, "timeToFirstToolCallMs": 39059 }, "metrics": { "process": { "sampleCount": 198, "peakRssMb": 11991, "peakCpuPercent": 121.8, "firstAtMs": 0, "lastAtMs": 989208 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 198, "peakUsedMb": 69199, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 44277, "totalOutputTokens": 20841, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 20, "todayTokensIn": 490674, "todayTokensCached": 487122, "todayTokensOut": 10103, "todayCost": 0, "totalTurns": 20, "totalTokensIn": 490674, "totalTokensCached": 487122, "totalTokensOut": 10103, "totalCost": 0, "medianOutputTokensPerSec": 36.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.3, "turns": 20 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 28611, "cachedInputTokens": 28273, "outputTokens": 2514, "cost": 0, "durationMs": 69233, "outputTokensPerSec": 36.5534939880747, "at": "2026-11-02T14:52:56.341Z" }, "lastUpdated": "2026-11-02T14:52:56.341Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T14:56:29.801Z" } }, "source": "engine-log" }, "derived": { "durationMs": 993003, "meanTokensPerSec": 36, "genTokensPerSec": 36, "genTokens": 20841, "promptTokensPerSec": null, "peakRssMb": 11991, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 490674, "output": 10103 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 475, "sampleCount": 23, "structuredSampleCount": 0, "firstTokenSampleCount": 23, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-author-params", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 3 }, "trialId": "craftbook-author-params-gemma4-12b-q4-2026-11-02T14-56-53-892Z-ag8g", "success": false, "includedInCapability": true, "durationMs": 1028316, "timing": { "startedAt": "2026-11-02T14:56:53.893Z", "finishedAt": "2026-11-02T15:14:02.209Z", "timeToFirstArtifactMs": 106061, "timeToLastArtifactWriteMs": 1018860, "timeToFirstTokenMs": 15301, "firstTurnTtftMs": 13403, "timeToFirstToolCallMs": 19612 }, "metrics": { "process": { "sampleCount": 204, "peakRssMb": 28275, "peakCpuPercent": 117.6, "firstAtMs": 0, "lastAtMs": 1019690 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 204, "peakUsedMb": 84378, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 109977, "totalOutputTokens": 24504, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 280467, "todayTokensCached": 255540, "todayTokensOut": 5234, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 280467, "totalTokensCached": 255540, "totalTokensOut": 5234, "totalCost": 0, "medianOutputTokensPerSec": 79.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 79.6, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17748, "cachedInputTokens": 13680, "outputTokens": 208, "cost": 0, "durationMs": 20441, "outputTokensPerSec": 103.92625352884858, "at": "2026-11-02T15:13:52.753Z" }, "lastUpdated": "2026-11-02T15:13:52.753Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T15:13:57.944Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1023001, "meanTokensPerSec": 60.15, "genTokensPerSec": 60.15, "genTokens": 24504, "promptTokensPerSec": 1063.61, "peakRssMb": 28275, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 280467, "output": 5234 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-author-params", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 3 }, "trialId": "craftbook-author-params-mlx-qwen3-8-27b-q4-2026-11-02T15-14-24-526Z-k7mx", "success": false, "includedInCapability": false, "durationMs": 4899421, "timing": { "startedAt": "2026-11-02T15:14:24.527Z", "finishedAt": "2026-11-02T16:36:03.948Z", "timeToFirstArtifactMs": 3853038, "timeToLastArtifactWriteMs": 3853038, "timeToFirstTokenMs": 47790, "firstTurnTtftMs": 46113, "timeToFirstToolCallMs": 55321 }, "metrics": { "process": { "sampleCount": 975, "peakRssMb": 19045, "peakCpuPercent": 188.3, "firstAtMs": 0, "lastAtMs": 4891839 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 975, "peakUsedMb": 86699, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 71547, "totalOutputTokens": 81671, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 139289, "todayTokensCached": 136862, "todayTokensOut": 10040, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 139289, "totalTokensCached": 136862, "totalTokensOut": 10040, "totalCost": 0, "medianOutputTokensPerSec": 11.6, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 11.6, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 11131, "cachedInputTokens": 11044, "outputTokens": 1185, "cost": 0, "durationMs": 284482, "outputTokensPerSec": 24.221569198051405, "at": "2026-11-02T16:18:37.565Z" }, "lastUpdated": "2026-11-02T16:18:37.565Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T16:35:59.469Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4893875, "meanTokensPerSec": 12, "genTokensPerSec": 12, "genTokens": 81671, "promptTokensPerSec": null, "peakRssMb": 19045, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 139289, "output": 10040 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 261.5, "sampleCount": 8, "structuredSampleCount": 0, "firstTokenSampleCount": 8, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-author-params", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 3 }, "trialId": "craftbook-author-params-qwen3-8-27b-q4-2026-11-02T17-32-57-489Z-vm07", "success": true, "includedInCapability": true, "durationMs": 1316565, "timing": { "startedAt": "2026-11-02T17:32:57.489Z", "finishedAt": "2026-11-02T17:54:54.054Z", "timeToFirstArtifactMs": 702721, "timeToLastArtifactWriteMs": 1310325, "timeToFirstTokenMs": 40153, "firstTurnTtftMs": 38439, "timeToFirstToolCallMs": 49220 }, "metrics": { "process": { "sampleCount": 262, "peakRssMb": 47328, "peakCpuPercent": 118.6, "firstAtMs": 0, "lastAtMs": 1309272 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 262, "peakUsedMb": 104202, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 179548, "totalOutputTokens": 23365, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 118081, "todayTokensCached": 67264, "todayTokensOut": 1244, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 118081, "totalTokensCached": 67264, "totalTokensOut": 1244, "totalCost": 0, "medianOutputTokensPerSec": 31.3, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 31.3, "turns": 5 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 9892, "cachedInputTokens": 9722, "outputTokens": 186, "cost": 0, "durationMs": 239710, "outputTokensPerSec": 55.62193440460177, "at": "2026-11-02T17:53:29.037Z" }, "lastUpdated": "2026-11-02T17:53:29.037Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T17:54:52.008Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1313571, "meanTokensPerSec": 28.86, "genTokensPerSec": 28.86, "genTokens": 23365, "promptTokensPerSec": 396.78, "peakRssMb": 47328, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 118081, "output": 1244 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-author-gate-script", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 3 }, "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-11-02T17-55-19-914Z-qa0i", "success": false, "includedInCapability": true, "durationMs": 865127, "timing": { "startedAt": "2026-11-02T17:55:19.915Z", "finishedAt": "2026-11-02T18:09:45.042Z", "timeToFirstArtifactMs": 84722, "timeToLastArtifactWriteMs": 854656, "timeToFirstTokenMs": 17708, "firstTurnTtftMs": 15859, "timeToFirstToolCallMs": 25772 }, "metrics": { "process": { "sampleCount": 171, "peakRssMb": 29414, "peakCpuPercent": 132.6, "firstAtMs": 0, "lastAtMs": 854129 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 171, "peakUsedMb": 85183, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 90349, "totalOutputTokens": 19696, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 19, "todayTokensIn": 353860, "todayTokensCached": 322273, "todayTokensOut": 5654, "todayCost": 0, "totalTurns": 19, "totalTokensIn": 353860, "totalTokensCached": 322273, "totalTokensOut": 5654, "totalCost": 0, "medianOutputTokensPerSec": 70.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 70.9, "turns": 19 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14593, "cachedInputTokens": 14270, "outputTokens": 261, "cost": 0, "durationMs": 11942, "outputTokensPerSec": 102.93222229876156, "at": "2026-11-02T18:09:34.575Z" }, "lastUpdated": "2026-11-02T18:09:34.575Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T18:09:38.158Z" } }, "source": "engine-log" }, "derived": { "durationMs": 857193, "meanTokensPerSec": 64.97, "genTokensPerSec": 64.97, "genTokens": 19696, "promptTokensPerSec": 1043.75, "peakRssMb": 29414, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 353860, "output": 5654 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-author-gate-script", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 3 }, "trialId": "craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-11-02T18-10-06-901Z-pic2", "success": false, "includedInCapability": true, "durationMs": 1454598, "timing": { "startedAt": "2026-11-02T18:10:06.902Z", "finishedAt": "2026-11-02T18:34:21.500Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 41471, "firstTurnTtftMs": 39800, "timeToFirstToolCallMs": 52050 }, "metrics": { "process": { "sampleCount": 289, "peakRssMb": 18084, "peakCpuPercent": 120.2, "firstAtMs": 0, "lastAtMs": 1444764 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 289, "peakUsedMb": 87593, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 90261, "totalOutputTokens": 17989, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T18:34:16.032Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1448060, "meanTokensPerSec": 15, "genTokensPerSec": 15, "genTokens": 17989, "promptTokensPerSec": null, "peakRssMb": 18084, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 411.5, "sampleCount": 2, "structuredSampleCount": 0, "firstTokenSampleCount": 2, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-author-gate-script", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 3 }, "trialId": "craftbook-author-gate-script-qwen3-8-27b-q4-2026-11-02T18-34-43-012Z-wcrx", "success": false, "includedInCapability": true, "durationMs": 2028955, "timing": { "startedAt": "2026-11-02T18:34:43.014Z", "finishedAt": "2026-11-02T19:08:31.969Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 49821, "firstTurnTtftMs": 47982, "timeToFirstToolCallMs": 53562 }, "metrics": { "process": { "sampleCount": 404, "peakRssMb": 47746, "peakCpuPercent": 115.8, "firstAtMs": 0, "lastAtMs": 2021612 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 404, "peakUsedMb": 105168, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 226819, "totalOutputTokens": 16045, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 109855, "todayTokensCached": 0, "todayTokensOut": 1298, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 109855, "totalTokensCached": 0, "totalTokensOut": 1298, "totalCost": 0, "medianOutputTokensPerSec": 11.7, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 11.7, "turns": 1 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 109855, "outputTokens": 1298, "cost": 0, "durationMs": 1995032, "outputTokensPerSec": 11.654141489670105, "at": "2026-11-02T19:07:59.885Z" }, "lastUpdated": "2026-11-02T19:07:59.885Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T19:08:29.729Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2025663, "meanTokensPerSec": 15.77, "genTokensPerSec": 15.77, "genTokens": 16045, "promptTokensPerSec": 236.28, "peakRssMb": 47746, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 109855, "output": 1298 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-author-gate-script", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 3 }, "trialId": "craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-11-02T19-08-53-661Z-9l03", "success": false, "includedInCapability": true, "durationMs": 1566256, "timing": { "startedAt": "2026-11-02T19:08:53.662Z", "finishedAt": "2026-11-02T19:34:59.918Z", "timeToFirstArtifactMs": 833343, "timeToLastArtifactWriteMs": 1560312, "timeToFirstTokenMs": 20191, "firstTurnTtftMs": 18637, "timeToFirstToolCallMs": 38064 }, "metrics": { "process": { "sampleCount": 311, "peakRssMb": 13463, "peakCpuPercent": 118, "firstAtMs": 0, "lastAtMs": 1556444 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 311, "peakUsedMb": 69895, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 166152, "totalOutputTokens": 36023, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 34, "todayTokensIn": 545289, "todayTokensCached": 511621, "todayTokensOut": 12445, "todayCost": 0, "totalTurns": 34, "totalTokensIn": 545289, "totalTokensCached": 511621, "totalTokensOut": 12445, "totalCost": 0, "medianOutputTokensPerSec": 36.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 36.7, "turns": 34 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17288, "cachedInputTokens": 17052, "outputTokens": 83, "cost": 0, "durationMs": 63152, "outputTokensPerSec": 37.822411555073536, "at": "2026-11-02T19:34:53.973Z" }, "lastUpdated": "2026-11-02T19:34:53.973Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T19:34:56.098Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1561481, "meanTokensPerSec": 35, "genTokensPerSec": 35, "genTokens": 36023, "promptTokensPerSec": null, "peakRssMb": 13463, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 545289, "output": 12445 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 524.5, "sampleCount": 36, "structuredSampleCount": 0, "firstTokenSampleCount": 36, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-mlx-complex-work-craftbook-author-fanout", "provider": "mlx", "model": "qwen3.8-27b-q4", "targetId": "qwen-mlx", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 3 }, "trialId": "craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-11-02T19-35-24-686Z-nnsw", "success": false, "includedInCapability": true, "durationMs": 1664787, "timing": { "startedAt": "2026-11-02T19:35:24.686Z", "finishedAt": "2026-11-02T20:03:09.473Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 41608, "firstTurnTtftMs": 39838, "timeToFirstToolCallMs": 57371 }, "metrics": { "process": { "sampleCount": 331, "peakRssMb": 18010, "peakCpuPercent": 148.6, "firstAtMs": 0, "lastAtMs": 1658711 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 331, "peakUsedMb": 85379, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 34046, "totalOutputTokens": 16764, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T20:03:05.605Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1659749, "meanTokensPerSec": 11, "genTokensPerSec": 11, "genTokens": 16764, "promptTokensPerSec": null, "peakRssMb": 18010, "peakGpuUtilPercent": null, "billing": null } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 543, "sampleCount": 1, "structuredSampleCount": 0, "firstTokenSampleCount": 1, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-qwen-llamacpp-complex-work-craftbook-author-fanout", "provider": "llama-cpp", "model": "qwen3.8-27b-q4", "targetId": "qwen-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 3 }, "trialId": "craftbook-author-fanout-qwen3-8-27b-q4-2026-11-02T20-03-31-159Z-a48x", "success": false, "includedInCapability": true, "durationMs": 3587819, "timing": { "startedAt": "2026-11-02T20:03:31.161Z", "finishedAt": "2026-11-02T21:03:18.980Z", "timeToFirstArtifactMs": 1311653, "timeToLastArtifactWriteMs": 1670460, "timeToFirstTokenMs": 50982, "firstTurnTtftMs": 49232, "timeToFirstToolCallMs": 55551 }, "metrics": { "process": { "sampleCount": 714, "peakRssMb": 49122, "peakCpuPercent": 119.4, "firstAtMs": 0, "lastAtMs": 3576620 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 714, "peakUsedMb": 106083, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 312721, "totalOutputTokens": 34386, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 179581, "todayTokensCached": 100898, "todayTokensOut": 2022, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 179581, "totalTokensCached": 100898, "totalTokensOut": 2022, "totalCost": 0, "medianOutputTokensPerSec": 15.4, "modelSpeeds": [ { "model": "qwen3.8-27b-q4", "medianOutputTokensPerSec": 15.4, "turns": 4 } ], "lastTurn": { "model": "qwen3.8-27b-q4", "inputTokens": 65142, "outputTokens": 720, "cost": 0, "durationMs": 554847, "outputTokensPerSec": 15.73044656812523, "at": "2026-11-02T20:37:25.945Z" }, "lastUpdated": "2026-11-02T20:37:25.945Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T21:03:13.028Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3580921, "meanTokensPerSec": 20.17, "genTokensPerSec": 20.17, "genTokens": 34386, "promptTokensPerSec": 288.77, "peakRssMb": 49122, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 179581, "output": 2022 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "qwen3.8-27b-q4" } }, { "cell": { "id": "r3-gemma-mlx-complex-work-craftbook-author-fanout", "provider": "mlx", "model": "gemma4-12b-q4", "targetId": "gemma-mlx", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 3 }, "trialId": "craftbook-author-fanout-mlx-gemma4-12b-q4-2026-11-02T21-03-44-160Z-lsom", "success": false, "includedInCapability": true, "durationMs": 937892, "timing": { "startedAt": "2026-11-02T21:03:44.161Z", "finishedAt": "2026-11-02T21:19:22.053Z", "timeToFirstArtifactMs": 704617, "timeToLastArtifactWriteMs": 933824, "timeToFirstTokenMs": 21231, "firstTurnTtftMs": 19562, "timeToFirstToolCallMs": 51881 }, "metrics": { "process": { "sampleCount": 187, "peakRssMb": 11128, "peakCpuPercent": 148.5, "firstAtMs": 0, "lastAtMs": 934413 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 187, "peakUsedMb": 68683, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 89308, "totalOutputTokens": 22270, "totalCostUsd": 0, "rawProviders": { "mlx": { "quotaBuckets": [], "todayTurns": 21, "todayTokensIn": 387001, "todayTokensCached": 369423, "todayTokensOut": 3908, "todayCost": 0, "totalTurns": 21, "totalTokensIn": 387001, "totalTokensCached": 369423, "totalTokensOut": 3908, "totalCost": 0, "medianOutputTokensPerSec": 35, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 35, "turns": 21 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8724, "cachedInputTokens": 1253, "outputTokens": 97, "cost": 0, "durationMs": 18569, "outputTokensPerSec": 35.710265877394356, "at": "2026-11-02T21:19:17.988Z" }, "lastUpdated": "2026-11-02T21:19:17.988Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T21:19:19.919Z" } }, "source": "engine-log" }, "derived": { "durationMs": 934699, "meanTokensPerSec": 33, "genTokensPerSec": 33, "genTokens": 22270, "promptTokensPerSec": null, "peakRssMb": 11128, "peakGpuUtilPercent": null, "billing": { "provider": "mlx", "limitedQuota": [], "tokens": { "input": 387001, "output": 3908 } } } }, "observedMlxPrefill": { "source": "first-token-detail", "aggregation": "median-observed-events", "tokensPerSec": 506, "sampleCount": 21, "structuredSampleCount": 0, "firstTokenSampleCount": 21, "malformedLineCount": 0, "duplicateLineCount": 0 }, "engineContext": null }, { "cell": { "id": "r3-gemma-llamacpp-complex-work-craftbook-author-fanout", "provider": "llama-cpp", "model": "gemma4-12b-q4", "targetId": "gemma-llamacpp", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 3 }, "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-11-02T21-19-48-250Z-3yci", "success": false, "includedInCapability": true, "durationMs": 1054660, "timing": { "startedAt": "2026-11-02T21:19:48.251Z", "finishedAt": "2026-11-02T21:37:22.911Z", "timeToFirstArtifactMs": 446590, "timeToLastArtifactWriteMs": 1048128, "timeToFirstTokenMs": 20505, "firstTurnTtftMs": 18528, "timeToFirstToolCallMs": 30605 }, "metrics": { "process": { "sampleCount": 209, "peakRssMb": 27573, "peakCpuPercent": 112.9, "firstAtMs": 0, "lastAtMs": 1044615 }, "gpu": { "available": false, "sampleCount": 0, "peakUtilPercent": 0, "peakMemUsedMb": null, "memTotalMb": null, "memoryModel": "unknown" }, "systemMemory": { "sampleCount": 209, "peakUsedMb": 84382, "totalMb": 131072 }, "usage": { "available": true, "totalInputTokens": 130371, "totalOutputTokens": 24097, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 19, "todayTokensIn": 354953, "todayTokensCached": 345256, "todayTokensOut": 8670, "todayCost": 0, "totalTurns": 19, "totalTokensIn": 354953, "totalTokensCached": 345256, "totalTokensOut": 8670, "totalCost": 0, "medianOutputTokensPerSec": 52.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 52.7, "turns": 19 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11887, "cachedInputTokens": 11744, "outputTokens": 35, "cost": 0, "durationMs": 73110, "outputTokensPerSec": 127.8632915771937, "at": "2026-11-02T21:36:42.148Z" }, "lastUpdated": "2026-11-02T21:36:42.148Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-02T21:37:17.551Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1048160, "meanTokensPerSec": 54.2, "genTokensPerSec": 54.2, "genTokens": 24097, "promptTokensPerSec": 1034.4, "peakRssMb": 27573, "peakGpuUtilPercent": null, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 354953, "output": 8670 } } } }, "observedMlxPrefill": null, "engineContext": { "launches": 1, "grantedPerSlotTokens": 262144, "totalTokens": 262144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" } } ] }