{
  "note": "Completed MLX spec-request counts supplement pulse totals; cancelled/unlogged requests remain unknown. They are not a replacement for exact total usage.",
  "mlxPrefillRecovery": "mlx-prefill-recovery.json",
  "trials": [
    {
      "id": "r1-qwen-mlx-core-tictactoe",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "tictactoe",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-11-03T07-06-18-943Z-cp0v",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 115.222,
      "firstTokenSeconds": 21.027,
      "firstToolSeconds": 35.249,
      "firstArtifactSeconds": 105.41,
      "decodeTokensPerSecond": 45,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 722,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 16743,
      "outputTokensReported": 2879,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 11914,
      "apiOutputTokensReported": 505,
      "peakSubtreeRssGiB": 17.6357421875,
      "peakSystemUsedGiB": 109.0244140625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 4,
      "completedSpecRequestGeneratedTokens": 3267,
      "specAcceptedTokensPerRound": {
        "n": 4,
        "median": 2.455,
        "mean": 2.4825,
        "min": 2.24,
        "max": 2.78
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 6,
        "functionalCompleteness": 7,
        "codeQuality": 7,
        "polish": 6
      }
    },
    {
      "id": "r1-qwen-llamacpp-core-tictactoe",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "tictactoe",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-tictactoe/tictactoe/tictactoe-qwen3-8-27b-q4-2026-11-03T07-09-31-185Z-spl5",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 8.8,
      "durationSeconds": 77.133,
      "firstTokenSeconds": 18.646,
      "firstToolSeconds": 26.712,
      "firstArtifactSeconds": 68.115,
      "decodeTokensPerSecond": 50.85,
      "prefillTokensPerSecond": 630.37,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 11581,
      "outputTokensReported": 1736,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 7330,
      "apiOutputTokensReported": 168,
      "peakSubtreeRssGiB": 37.263671875,
      "peakSystemUsedGiB": 116.46484375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 5,
        "functionalCompleteness": 6,
        "codeQuality": 6,
        "polish": 4
      }
    },
    {
      "id": "r1-gemma-mlx-core-tictactoe",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "tictactoe",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-11-03T07-12-05-257Z-ue03",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 74.245,
      "firstTokenSeconds": 13.971,
      "firstToolSeconds": 20.201,
      "firstArtifactSeconds": 32.23,
      "decodeTokensPerSecond": 69,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1669,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 3,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 26767,
      "outputTokensReported": 2132,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 16967,
      "apiOutputTokensReported": 189,
      "peakSubtreeRssGiB": 10.1025390625,
      "peakSystemUsedGiB": 92.6787109375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 5,
      "completedSpecRequestGeneratedTokens": 2549,
      "specAcceptedTokensPerRound": {
        "n": 5,
        "median": 2.89,
        "mean": 3.038,
        "min": 2.57,
        "max": 3.6
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 5,
        "functionalCompleteness": 5,
        "codeQuality": 5,
        "polish": 3
      }
    },
    {
      "id": "r1-gemma-llamacpp-core-tictactoe",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "tictactoe",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-tictactoe/tictactoe/tictactoe-gemma4-12b-q4-2026-11-03T07-14-32-380Z-3m76",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 41.138,
      "firstTokenSeconds": 9.672,
      "firstToolSeconds": 13.206,
      "firstArtifactSeconds": 35.247,
      "decodeTokensPerSecond": 123.39,
      "prefillTokensPerSecond": 1340.54,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 13567,
      "outputTokensReported": 1771,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 6952,
      "apiOutputTokensReported": 43,
      "peakSubtreeRssGiB": 17.228515625,
      "peakSystemUsedGiB": 96.2578125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 5,
        "functionalCompleteness": 5,
        "codeQuality": 5,
        "polish": 3
      }
    },
    {
      "id": "r1-qwen-llamacpp-core-petshop",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "petshop",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-petshop/petshop/petshop-qwen3-8-27b-q4-2026-11-03T07-16-15-159Z-wkjx",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 1183.198,
      "firstTokenSeconds": 18.739,
      "firstToolSeconds": 33.286,
      "firstArtifactSeconds": 307.249,
      "decodeTokensPerSecond": 33.37,
      "prefillTokensPerSecond": 443.73,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 59898,
      "outputTokensReported": 32422,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 48321,
      "apiOutputTokensReported": 2952,
      "peakSubtreeRssGiB": 51.01171875,
      "peakSystemUsedGiB": 127.94140625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 6,
        "functionalCompleteness": 5,
        "codeQuality": 5,
        "polish": 4
      }
    },
    {
      "id": "r1-gemma-mlx-core-petshop",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "petshop",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-11-03T07-36-55-804Z-c7vf",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 8,
      "durationSeconds": 294.838,
      "firstTokenSeconds": 17.136,
      "firstToolSeconds": 26.643,
      "firstArtifactSeconds": 70.918,
      "decodeTokensPerSecond": 57,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1108,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 8,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 93237,
      "outputTokensReported": 6893,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 74553,
      "apiOutputTokensReported": 2972,
      "peakSubtreeRssGiB": 18.0615234375,
      "peakSystemUsedGiB": 95.46484375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 18,
      "completedSpecRequestGeneratedTokens": 7468,
      "specAcceptedTokensPerRound": {
        "n": 18,
        "median": 3.1900000000000004,
        "mean": 3.2583333333333333,
        "min": 2.47,
        "max": 3.85
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 5,
        "functionalCompleteness": 4,
        "codeQuality": 4,
        "polish": 3
      }
    },
    {
      "id": "r1-gemma-llamacpp-core-petshop",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "petshop",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-petshop/petshop/petshop-gemma4-12b-q4-2026-11-03T07-42-51-928Z-5sas",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 8,
      "durationSeconds": 277.289,
      "firstTokenSeconds": 10.977,
      "firstToolSeconds": 22.962,
      "firstArtifactSeconds": 62.237,
      "decodeTokensPerSecond": 81.72,
      "prefillTokensPerSecond": 1101.53,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 79654,
      "outputTokensReported": 11011,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 64945,
      "apiOutputTokensReported": 2815,
      "peakSubtreeRssGiB": 28.44140625,
      "peakSystemUsedGiB": 107.5546875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 6,
        "functionalCompleteness": 4,
        "codeQuality": 5,
        "polish": 4
      }
    },
    {
      "id": "r1-qwen-mlx-core-petshop",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "petshop",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-11-03T07-48-27-109Z-vg4t",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": false,
      "failureClass": "model",
      "fixedComposite": 1.7,
      "durationSeconds": 2866.983,
      "firstTokenSeconds": 29.027,
      "firstToolSeconds": 41.961,
      "firstArtifactSeconds": 673.015,
      "decodeTokensPerSecond": 26,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 493,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 17,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 166105,
      "outputTokensReported": 60189,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 171401,
      "apiOutputTokensReported": 17477,
      "peakSubtreeRssGiB": 20.015625,
      "peakSystemUsedGiB": 104.19921875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 47,
      "completedSpecRequestGeneratedTokens": 57271,
      "specAcceptedTokensPerRound": {
        "n": 47,
        "median": 2.33,
        "mean": 2.411914893617021,
        "min": 1.9,
        "max": 3
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 6,
        "functionalCompleteness": 5,
        "codeQuality": 6,
        "polish": 5
      }
    },
    {
      "id": "r1-gemma-mlx-core-tankcombat",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "tankcombat",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-11-03T08-36-58-686Z-po7v",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 96.905,
      "firstTokenSeconds": 17.461,
      "firstToolSeconds": 37.633,
      "firstArtifactSeconds": 87.514,
      "decodeTokensPerSecond": 65,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1512,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 25428,
      "outputTokensReported": 2727,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 10721,
      "apiOutputTokensReported": 24,
      "peakSubtreeRssGiB": 10.0693359375,
      "peakSystemUsedGiB": 91.708984375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 4,
      "completedSpecRequestGeneratedTokens": 3188,
      "specAcceptedTokensPerRound": {
        "n": 4,
        "median": 3.13,
        "mean": 3.1325000000000003,
        "min": 2.67,
        "max": 3.6
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 4,
        "functionalCompleteness": 4,
        "codeQuality": 3,
        "polish": 2
      }
    },
    {
      "id": "r1-gemma-llamacpp-core-tankcombat",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "tankcombat",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-tankcombat/tankcombat/tankcombat-gemma4-12b-q4-2026-11-03T08-39-38-780Z-qy7d",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 58.752,
      "firstTokenSeconds": 9.325,
      "firstToolSeconds": 16.385,
      "firstArtifactSeconds": 48.085,
      "decodeTokensPerSecond": 112.93,
      "prefillTokensPerSecond": 1519.39,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 13578,
      "outputTokensReported": 2945,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 13570,
      "apiOutputTokensReported": 2244,
      "peakSubtreeRssGiB": 17.0947265625,
      "peakSystemUsedGiB": 93.7431640625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 5,
        "functionalCompleteness": 5,
        "codeQuality": 4,
        "polish": 3
      }
    },
    {
      "id": "r1-qwen-mlx-core-tankcombat",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "tankcombat",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-11-03T08-41-33-861Z-4y65",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 373.103,
      "firstTokenSeconds": 22.335,
      "firstToolSeconds": 31.463,
      "firstArtifactSeconds": 360.864,
      "decodeTokensPerSecond": 34,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 752,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 15105,
      "outputTokensReported": 10286,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 12251,
      "apiOutputTokensReported": 783,
      "peakSubtreeRssGiB": 17.4990234375,
      "peakSystemUsedGiB": 107.048828125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 5,
      "completedSpecRequestGeneratedTokens": 10838,
      "specAcceptedTokensPerRound": {
        "n": 5,
        "median": 2.25,
        "mean": 2.344,
        "min": 2.21,
        "max": 2.53
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 7,
        "functionalCompleteness": 6,
        "codeQuality": 5,
        "polish": 7
      }
    },
    {
      "id": "r1-qwen-llamacpp-core-tankcombat",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "tankcombat",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-tankcombat/tankcombat/tankcombat-qwen3-8-27b-q4-2026-11-03T08-48-40-909Z-bu3h",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 201.463,
      "firstTokenSeconds": 23.874,
      "firstToolSeconds": 38.541,
      "firstArtifactSeconds": 195.404,
      "decodeTokensPerSecond": 35.85,
      "prefillTokensPerSecond": 449.19,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 14607,
      "outputTokensReported": 5406,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 15639,
      "apiOutputTokensReported": 444,
      "peakSubtreeRssGiB": 38.9541015625,
      "peakSystemUsedGiB": 115.087890625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "visualQuality": 5,
        "functionalCompleteness": 4,
        "codeQuality": 4,
        "polish": 3
      }
    },
    {
      "id": "r1-gemma-llamacpp-core-schema-migration",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "schema-migration",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-schema-migration/schema-migration/schema-migration-gemma4-12b-q4-2026-11-03T08-53-03-481Z-wdkd",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 217.355,
      "firstTokenSeconds": 9.81,
      "firstToolSeconds": 14.091,
      "firstArtifactSeconds": 25.063,
      "decodeTokensPerSecond": 75.81,
      "prefillTokensPerSecond": 1041.15,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 97867,
      "outputTokensReported": 5617,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 103127,
      "apiOutputTokensReported": 4161,
      "peakSubtreeRssGiB": 25.20703125,
      "peakSystemUsedGiB": 95.9013671875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-schema-migration",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "schema-migration",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-11-03T08-57-15-039Z-3926",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 194.093,
      "firstTokenSeconds": 15.406,
      "firstToolSeconds": 17.408,
      "firstArtifactSeconds": 30.774,
      "decodeTokensPerSecond": 36,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 574,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 7,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 39198,
      "outputTokensReported": 2518,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 38086,
      "apiOutputTokensReported": 2326,
      "peakSubtreeRssGiB": 17.1513671875,
      "peakSystemUsedGiB": 109.013671875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 13,
      "completedSpecRequestGeneratedTokens": 3481,
      "specAcceptedTokensPerRound": {
        "n": 13,
        "median": 2.8,
        "mean": 2.8030769230769232,
        "min": 2.52,
        "max": 2.97
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-schema-migration",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "schema-migration",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-schema-migration/schema-migration/schema-migration-qwen3-8-27b-q4-2026-11-03T09-01-08-005Z-pidy",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 217.805,
      "firstTokenSeconds": 17.392,
      "firstToolSeconds": 19.195,
      "firstArtifactSeconds": 41.076,
      "decodeTokensPerSecond": 35.46,
      "prefillTokensPerSecond": 435.81,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 45633,
      "outputTokensReported": 3151,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 27047,
      "apiOutputTokensReported": 1538,
      "peakSubtreeRssGiB": 42.955078125,
      "peakSystemUsedGiB": 115.1083984375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-schema-migration",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "schema-migration",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-11-03T09-05-20-192Z-fn5m",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9,
      "durationSeconds": 350.23,
      "firstTokenSeconds": 12.836,
      "firstToolSeconds": 22.275,
      "firstArtifactSeconds": 29.02,
      "decodeTokensPerSecond": 47,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1073.5,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 12,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 149406,
      "outputTokensReported": 5169,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 124605,
      "apiOutputTokensReported": 3287,
      "peakSubtreeRssGiB": 10.966796875,
      "peakSystemUsedGiB": 90.8818359375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 27,
      "completedSpecRequestGeneratedTokens": 8579,
      "specAcceptedTokensPerRound": {
        "n": 27,
        "median": 3.29,
        "mean": 3.3051851851851852,
        "min": 2.5,
        "max": 4
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-failing-tests-spec",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "failing-tests-spec",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-11-03T09-11-43-635Z-kqkp",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 69.257,
      "firstTokenSeconds": 14.578,
      "firstToolSeconds": 16.519,
      "firstArtifactSeconds": 37.092,
      "decodeTokensPerSecond": 38,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 596.5,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 20265,
      "outputTokensReported": 391,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 12919,
      "apiOutputTokensReported": 525,
      "peakSubtreeRssGiB": 17.208984375,
      "peakSystemUsedGiB": 107.2431640625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 5,
      "completedSpecRequestGeneratedTokens": 665,
      "specAcceptedTokensPerRound": {
        "n": 5,
        "median": 2.96,
        "mean": 2.886,
        "min": 2.64,
        "max": 3
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-failing-tests-spec",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "failing-tests-spec",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-qwen3-8-27b-q4-2026-11-03T09-13-26-528Z-kf1k",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 39.891,
      "firstTokenSeconds": 15.755,
      "firstToolSeconds": 16.675,
      "firstArtifactSeconds": 32.755,
      "decodeTokensPerSecond": 46.69,
      "prefillTokensPerSecond": 463.53,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 6265,
      "outputTokensReported": 531,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 6325,
      "apiOutputTokensReported": 470,
      "peakSubtreeRssGiB": 35.8232421875,
      "peakSystemUsedGiB": 110.2685546875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-failing-tests-spec",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "failing-tests-spec",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-11-03T09-14-39-926Z-zx8u",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 49.045,
      "firstTokenSeconds": 10.632,
      "firstToolSeconds": 11.002,
      "firstArtifactSeconds": 42.553,
      "decodeTokensPerSecond": 80,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1963,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 18269,
      "outputTokensReported": 1220,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 7129,
      "apiOutputTokensReported": 703,
      "peakSubtreeRssGiB": 10.1357421875,
      "peakSystemUsedGiB": 91.68359375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 4,
      "completedSpecRequestGeneratedTokens": 1846,
      "specAcceptedTokensPerRound": {
        "n": 4,
        "median": 3.5700000000000003,
        "mean": 3.5675,
        "min": 3.19,
        "max": 3.94
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-failing-tests-spec",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "failing-tests-spec",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-11-03T09-16-02-754Z-h3d5",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 24.76,
      "firstTokenSeconds": 8.602,
      "firstToolSeconds": 8.715,
      "firstArtifactSeconds": 21.182,
      "decodeTokensPerSecond": 114.56,
      "prefillTokensPerSecond": 1385.83,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 13667,
      "outputTokensReported": 703,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 6622,
      "apiOutputTokensReported": 658,
      "peakSubtreeRssGiB": 15.7158203125,
      "peakSystemUsedGiB": 88.0908203125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-symptom-debug",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "symptom-debug",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-symptom-debug/symptom-debug/symptom-debug-qwen3-8-27b-q4-2026-11-03T09-17-00-257Z-rnyl",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 34.069,
      "firstTokenSeconds": 14.42,
      "firstToolSeconds": 15.485,
      "firstArtifactSeconds": 29.544,
      "decodeTokensPerSecond": 40.53,
      "prefillTokensPerSecond": 592.97,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 5400,
      "outputTokensReported": 510,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 35.994140625,
      "peakSystemUsedGiB": 110.529296875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-symptom-debug",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "symptom-debug",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-11-03T09-18-07-118Z-8fo4",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 186.9,
      "firstTokenSeconds": 11.169,
      "firstToolSeconds": 12.862,
      "firstArtifactSeconds": 178.274,
      "decodeTokensPerSecond": 67,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1373.5,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 20125,
      "outputTokensReported": 8772,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 21338,
      "apiOutputTokensReported": 8298,
      "peakSubtreeRssGiB": 9.8671875,
      "peakSystemUsedGiB": 90.861328125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 5,
      "completedSpecRequestGeneratedTokens": 9269,
      "specAcceptedTokensPerRound": {
        "n": 5,
        "median": 2.9,
        "mean": 2.926,
        "min": 2.39,
        "max": 3.57
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-symptom-debug",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "symptom-debug",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-symptom-debug/symptom-debug/symptom-debug-gemma4-12b-q4-2026-11-03T09-21-52-926Z-j457",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 165.059,
      "firstTokenSeconds": 10.161,
      "firstToolSeconds": 13.403,
      "firstArtifactSeconds": 159.309,
      "decodeTokensPerSecond": 65.4,
      "prefillTokensPerSecond": 893.16,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 21412,
      "outputTokensReported": 8486,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 21412,
      "apiOutputTokensReported": 8253,
      "peakSubtreeRssGiB": 15.64453125,
      "peakSystemUsedGiB": 86.25,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-symptom-debug",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "symptom-debug",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-11-03T09-25-13-163Z-p8hj",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 57.654,
      "firstTokenSeconds": 15.085,
      "firstToolSeconds": 16.682,
      "firstArtifactSeconds": 55.914,
      "decodeTokensPerSecond": 31,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 544,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 5566,
      "outputTokensReported": 1107,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 17.16015625,
      "peakSystemUsedGiB": 105.8759765625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 2,
      "completedSpecRequestGeneratedTokens": 1192,
      "specAcceptedTokensPerRound": {
        "n": 2,
        "median": 2.44,
        "mean": 2.44,
        "min": 2.32,
        "max": 2.56
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-data-wrangle",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "data-wrangle",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-11-03T09-26-43-638Z-lv3u",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": false,
      "failureClass": "model",
      "fixedComposite": 3.9,
      "durationSeconds": 645.556,
      "firstTokenSeconds": 12.54,
      "firstToolSeconds": 22.589,
      "firstArtifactSeconds": 45.954,
      "decodeTokensPerSecond": 55,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1046,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 7,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 116281,
      "outputTokensReported": 24659,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 98109,
      "apiOutputTokensReported": 18049,
      "peakSubtreeRssGiB": 10.43359375,
      "peakSystemUsedGiB": 90.5595703125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 13,
      "completedSpecRequestGeneratedTokens": 19949,
      "specAcceptedTokensPerRound": {
        "n": 13,
        "median": 3.43,
        "mean": 3.4692307692307693,
        "min": 3.07,
        "max": 4
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-data-wrangle",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "data-wrangle",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-data-wrangle/data-wrangle/data-wrangle-gemma4-12b-q4-2026-11-03T09-38-02-650Z-jx4j",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 85.336,
      "firstTokenSeconds": 10.199,
      "firstToolSeconds": 15.764,
      "firstArtifactSeconds": 36.409,
      "decodeTokensPerSecond": 89.09,
      "prefillTokensPerSecond": 853.2,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 12893,
      "outputTokensReported": 4982,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 18.83984375,
      "peakSystemUsedGiB": 90.08203125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-data-wrangle",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "data-wrangle",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-11-03T09-39-57-626Z-vm00",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 72.883,
      "firstTokenSeconds": 14.742,
      "firstToolSeconds": 16.116,
      "firstArtifactSeconds": 64.248,
      "decodeTokensPerSecond": 39,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 613,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 7385,
      "outputTokensReported": 1404,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 17.236328125,
      "peakSystemUsedGiB": 106.9228515625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 5,
      "completedSpecRequestGeneratedTokens": 1791,
      "specAcceptedTokensPerRound": {
        "n": 5,
        "median": 2.56,
        "mean": 2.52,
        "min": 2.03,
        "max": 2.83
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-data-wrangle",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "data-wrangle",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-data-wrangle/data-wrangle/data-wrangle-qwen3-8-27b-q4-2026-11-03T09-41-43-078Z-a829",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 81.983,
      "firstTokenSeconds": 16.472,
      "firstToolSeconds": 19.656,
      "firstArtifactSeconds": 62.136,
      "decodeTokensPerSecond": 38.19,
      "prefillTokensPerSecond": 448.78,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 8698,
      "outputTokensReported": 1689,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 36.833984375,
      "peakSystemUsedGiB": 109.8623046875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-incident-postmortem",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "incident-postmortem",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-incident-postmortem/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-11-03T09-43-38-956Z-najw",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 178.93,
      "firstTokenSeconds": 13.817,
      "firstToolSeconds": 18.123,
      "firstArtifactSeconds": 73.364,
      "decodeTokensPerSecond": 63.36,
      "prefillTokensPerSecond": 1088.71,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 44049,
      "outputTokensReported": 7323,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 43840,
      "apiOutputTokensReported": 3597,
      "peakSubtreeRssGiB": 22.49609375,
      "peakSystemUsedGiB": 93.9912109375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-incident-postmortem",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "incident-postmortem",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-11-03T09-47-14-688Z-rx7i",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 635.937,
      "firstTokenSeconds": 28.656,
      "firstToolSeconds": 39.572,
      "firstArtifactSeconds": 629.089,
      "decodeTokensPerSecond": 27,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 562,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 31945,
      "outputTokensReported": 13901,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 17.169921875,
      "peakSystemUsedGiB": 106.125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 6,
      "completedSpecRequestGeneratedTokens": 14145,
      "specAcceptedTokensPerRound": {
        "n": 6,
        "median": 2.4450000000000003,
        "mean": 2.4566666666666666,
        "min": 2.11,
        "max": 2.9
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-incident-postmortem",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "incident-postmortem",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-incident-postmortem/incident-postmortem/incident-postmortem-qwen3-8-27b-q4-2026-11-03T09-58-24-273Z-xunw",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 403.501,
      "firstTokenSeconds": 34.014,
      "firstToolSeconds": 38.757,
      "firstArtifactSeconds": 394.156,
      "decodeTokensPerSecond": 27.71,
      "prefillTokensPerSecond": 395.51,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 32488,
      "outputTokensReported": 8417,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 39.041015625,
      "peakSystemUsedGiB": 111.2353515625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-incident-postmortem",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "incident-postmortem",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-11-03T10-05-46-552Z-r8hi",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 213.711,
      "firstTokenSeconds": 17.602,
      "firstToolSeconds": 29.274,
      "firstArtifactSeconds": 146.55,
      "decodeTokensPerSecond": 51,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1169,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 3,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 94724,
      "outputTokensReported": 4633,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 51813,
      "apiOutputTokensReported": 1124,
      "peakSubtreeRssGiB": 9.9462890625,
      "peakSystemUsedGiB": 90.9775390625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 8,
      "completedSpecRequestGeneratedTokens": 5339,
      "specAcceptedTokensPerRound": {
        "n": 8,
        "median": 2.94,
        "mean": 3.01,
        "min": 2.5,
        "max": 3.83
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-ops-runbook-anomaly",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "ops-runbook-anomaly",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-11-03T10-09-52-914Z-l99e",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 118.413,
      "firstTokenSeconds": 27.346,
      "firstToolSeconds": 28.7,
      "firstArtifactSeconds": 100.999,
      "decodeTokensPerSecond": 32,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 572,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 12301,
      "outputTokensReported": 2435,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 12253,
      "apiOutputTokensReported": 410,
      "peakSubtreeRssGiB": 17.126953125,
      "peakSystemUsedGiB": 106.91796875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 4,
      "completedSpecRequestGeneratedTokens": 2743,
      "specAcceptedTokensPerRound": {
        "n": 4,
        "median": 2.55,
        "mean": 2.5475,
        "min": 2.23,
        "max": 2.86
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-ops-runbook-anomaly",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "ops-runbook-anomaly",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-qwen3-8-27b-q4-2026-11-03T10-12-25-083Z-knx6",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 114.087,
      "firstTokenSeconds": 32.502,
      "firstToolSeconds": 33.296,
      "firstArtifactSeconds": 42.567,
      "decodeTokensPerSecond": 37.2,
      "prefillTokensPerSecond": 457.65,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 28045,
      "outputTokensReported": 1170,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 18327,
      "apiOutputTokensReported": 345,
      "peakSubtreeRssGiB": 39.9560546875,
      "peakSystemUsedGiB": 112.5595703125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-ops-runbook-anomaly",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "ops-runbook-anomaly",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-11-03T10-14-52-305Z-v5t8",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 78.028,
      "firstTokenSeconds": 16.257,
      "firstToolSeconds": 22.704,
      "firstArtifactSeconds": 24.472,
      "decodeTokensPerSecond": 52,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1323,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 3,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 47031,
      "outputTokensReported": 281,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 26849,
      "apiOutputTokensReported": 239,
      "peakSubtreeRssGiB": 9.9580078125,
      "peakSystemUsedGiB": 91.7158203125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 20,
      "completedSpecRequestGeneratedTokens": 1326,
      "specAcceptedTokensPerRound": {
        "n": 20,
        "median": 3.44,
        "mean": 3.428,
        "min": 2.43,
        "max": 4
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-ops-runbook-anomaly",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "ops-runbook-anomaly",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-11-03T10-16-44-164Z-bubn",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 54.695,
      "firstTokenSeconds": 13.784,
      "firstToolSeconds": 17.455,
      "firstArtifactSeconds": 19.174,
      "decodeTokensPerSecond": 87.41,
      "prefillTokensPerSecond": 1121.83,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 31568,
      "outputTokensReported": 1079,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 18659,
      "apiOutputTokensReported": 253,
      "peakSubtreeRssGiB": 26.91796875,
      "peakSystemUsedGiB": 98.6875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-plan-and-estimate",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-qwen3-8-27b-q4-2026-11-03T10-18-12-831Z-yb8g",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 79.212,
      "firstTokenSeconds": 16.512,
      "firstToolSeconds": 17.906,
      "firstArtifactSeconds": 73.929,
      "decodeTokensPerSecond": 27.64,
      "prefillTokensPerSecond": 501.07,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 4749,
      "outputTokensReported": 1559,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 35.7880859375,
      "peakSystemUsedGiB": 110.15234375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-plan-and-estimate",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-11-03T10-20-05-274Z-ljgz",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 32.652,
      "firstTokenSeconds": 12.176,
      "firstToolSeconds": 21.109,
      "firstArtifactSeconds": 30.673,
      "decodeTokensPerSecond": 63,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1366,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 5699,
      "outputTokensReported": 636,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 9.986328125,
      "peakSystemUsedGiB": 91.025390625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 3,
      "completedSpecRequestGeneratedTokens": 1072,
      "specAcceptedTokensPerRound": {
        "n": 3,
        "median": 2.92,
        "mean": 2.8866666666666667,
        "min": 2.52,
        "max": 3.22
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-plan-and-estimate",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-11-03T10-21-11-588Z-y3zd",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 27.164,
      "firstTokenSeconds": 9.099,
      "firstToolSeconds": 13.077,
      "firstArtifactSeconds": 21.052,
      "decodeTokensPerSecond": 79.11,
      "prefillTokensPerSecond": 1200.21,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 5653,
      "outputTokensReported": 903,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 13.7041015625,
      "peakSystemUsedGiB": 86.1484375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-plan-and-estimate",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-03T10-22-12-154Z-fh19",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 67.801,
      "firstTokenSeconds": 13.117,
      "firstToolSeconds": 14.088,
      "firstArtifactSeconds": 62.633,
      "decodeTokensPerSecond": 40,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 801,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 4817,
      "outputTokensReported": 1770,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 17.1513671875,
      "peakSystemUsedGiB": 107.5537109375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 2,
      "completedSpecRequestGeneratedTokens": 1916,
      "specAcceptedTokensPerRound": {
        "n": 2,
        "median": 2.3,
        "mean": 2.3,
        "min": 2.07,
        "max": 2.53
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-core-conflict-synthesis",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "core",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-11-03T10-23-48-739Z-edca",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": false,
      "failureClass": "model",
      "fixedComposite": 6,
      "durationSeconds": 153.511,
      "firstTokenSeconds": 14.78,
      "firstToolSeconds": 20.453,
      "firstArtifactSeconds": 27.188,
      "decodeTokensPerSecond": 65,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1341.5,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 6,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 85527,
      "outputTokensReported": 2395,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 52510,
      "apiOutputTokensReported": 1623,
      "peakSubtreeRssGiB": 9.9404296875,
      "peakSystemUsedGiB": 91.7060546875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 39,
      "completedSpecRequestGeneratedTokens": 3836,
      "specAcceptedTokensPerRound": {
        "n": 39,
        "median": 3.67,
        "mean": 3.496923076923077,
        "min": 2.1,
        "max": 4
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-core-conflict-synthesis",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "core",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-11-03T10-26-57-342Z-klwn",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 137.924,
      "firstTokenSeconds": 15.358,
      "firstToolSeconds": 25.336,
      "firstArtifactSeconds": 32.708,
      "decodeTokensPerSecond": 70.34,
      "prefillTokensPerSecond": 1016.58,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 86702,
      "outputTokensReported": 2134,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 25874,
      "apiOutputTokensReported": 933,
      "peakSubtreeRssGiB": 19.6259765625,
      "peakSystemUsedGiB": 91.63671875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-core-conflict-synthesis",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "core",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-03T10-29-53-990Z-2z0b",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9,
      "durationSeconds": 490.082,
      "firstTokenSeconds": 27.889,
      "firstToolSeconds": 30.291,
      "firstArtifactSeconds": 168.135,
      "decodeTokensPerSecond": 27,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 553,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 46047,
      "outputTokensReported": 9437,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 12189,
      "apiOutputTokensReported": 199,
      "peakSubtreeRssGiB": 17.263671875,
      "peakSystemUsedGiB": 107.2626953125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 15,
      "completedSpecRequestGeneratedTokens": 10583,
      "specAcceptedTokensPerRound": {
        "n": 15,
        "median": 2.5,
        "mean": 2.570666666666667,
        "min": 2.1,
        "max": 3
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-core-conflict-synthesis",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "core",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-qwen3-8-27b-q4-2026-11-03T10-38-38-450Z-le5x",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 446.548,
      "firstTokenSeconds": 33.697,
      "firstToolSeconds": 35.753,
      "firstArtifactSeconds": 130.64,
      "decodeTokensPerSecond": 29.99,
      "prefillTokensPerSecond": 402.97,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 82423,
      "outputTokensReported": 6528,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 27351,
      "apiOutputTokensReported": 1619,
      "peakSubtreeRssGiB": 46.6455078125,
      "peakSystemUsedGiB": 119.671875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-productivity-constrained-comms",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "constrained-comms",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-constrained-comms/constrained-comms/constrained-comms-gemma4-12b-q4-2026-11-03T10-46-39-521Z-2jsj",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 33.205,
      "firstTokenSeconds": 14.812,
      "firstToolSeconds": 19.219,
      "firstArtifactSeconds": 28.048,
      "decodeTokensPerSecond": 79.55,
      "prefillTokensPerSecond": 1002.21,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 15538,
      "outputTokensReported": 600,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 5604,
      "apiOutputTokensReported": 243,
      "peakSubtreeRssGiB": 14.3701171875,
      "peakSystemUsedGiB": 86.875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-productivity-constrained-comms",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "constrained-comms",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-11-03T10-47-48-925Z-a2bm",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 68,
      "firstTokenSeconds": 21.493,
      "firstToolSeconds": 22.641,
      "firstArtifactSeconds": 31.788,
      "decodeTokensPerSecond": null,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 823,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 3,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 28882,
      "outputTokensReported": 6,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 11654,
      "apiOutputTokensReported": 485,
      "peakSubtreeRssGiB": 17.3076171875,
      "peakSystemUsedGiB": 108.38671875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 6,
      "completedSpecRequestGeneratedTokens": 599,
      "specAcceptedTokensPerRound": {
        "n": 6,
        "median": 2.79,
        "mean": 2.7716666666666665,
        "min": 2.51,
        "max": 3
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-productivity-constrained-comms",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "constrained-comms",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-constrained-comms/constrained-comms/constrained-comms-qwen3-8-27b-q4-2026-11-03T10-49-29-785Z-if9k",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 69.985,
      "firstTokenSeconds": 27.213,
      "firstToolSeconds": 27.944,
      "firstArtifactSeconds": 39.425,
      "decodeTokensPerSecond": 50.2,
      "prefillTokensPerSecond": 566.81,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 21457,
      "outputTokensReported": 680,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 15539,
      "apiOutputTokensReported": 577,
      "peakSubtreeRssGiB": 38.5556640625,
      "peakSystemUsedGiB": 112.0234375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-productivity-constrained-comms",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "constrained-comms",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-11-03T10-51-16-423Z-i847",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 42.596,
      "firstTokenSeconds": 15.825,
      "firstToolSeconds": 21.409,
      "firstArtifactSeconds": 28.887,
      "decodeTokensPerSecond": 66,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1409.5,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 21268,
      "outputTokensReported": 330,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 11336,
      "apiOutputTokensReported": 403,
      "peakSubtreeRssGiB": 9.8857421875,
      "peakSystemUsedGiB": 91.3662109375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 3,
      "completedSpecRequestGeneratedTokens": 773,
      "specAcceptedTokensPerRound": {
        "n": 3,
        "median": 3.2,
        "mean": 3.09,
        "min": 2.8,
        "max": 3.27
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-productivity-records-intake",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "records-intake",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-11-03T10-52-32-513Z-0ucf",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 57.679,
      "firstTokenSeconds": 14.178,
      "firstToolSeconds": 14.83,
      "firstArtifactSeconds": 55.647,
      "decodeTokensPerSecond": 49,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 692,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 5653,
      "outputTokensReported": 1466,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 17.2744140625,
      "peakSystemUsedGiB": 106.8701171875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 4,
      "completedSpecRequestGeneratedTokens": 1901,
      "specAcceptedTokensPerRound": {
        "n": 4,
        "median": 2.7800000000000002,
        "mean": 2.775,
        "min": 2.54,
        "max": 3
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-productivity-records-intake",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "records-intake",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-records-intake/records-intake/records-intake-qwen3-8-27b-q4-2026-11-03T10-54-02-338Z-6c2l",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 234.822,
      "firstTokenSeconds": 15.918,
      "firstToolSeconds": 17.5,
      "firstArtifactSeconds": 59.196,
      "decodeTokensPerSecond": 38.65,
      "prefillTokensPerSecond": 440.62,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 8236,
      "outputTokensReported": 7723,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 36.7939453125,
      "peakSystemUsedGiB": 110.7734375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-productivity-records-intake",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "records-intake",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-11-03T10-58-30-291Z-59ha",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 151.451,
      "firstTokenSeconds": 13.463,
      "firstToolSeconds": 32.41,
      "firstArtifactSeconds": 56.986,
      "decodeTokensPerSecond": 57,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1028,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 21368,
      "outputTokensReported": 4823,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 7572,
      "apiOutputTokensReported": 147,
      "peakSubtreeRssGiB": 9.943359375,
      "peakSystemUsedGiB": 92.087890625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 7,
      "completedSpecRequestGeneratedTokens": 5351,
      "specAcceptedTokensPerRound": {
        "n": 7,
        "median": 3.2,
        "mean": 3.177142857142857,
        "min": 2.72,
        "max": 3.62
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-productivity-records-intake",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "records-intake",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-records-intake/records-intake/records-intake-gemma4-12b-q4-2026-11-03T11-01-35-494Z-qh02",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 171.821,
      "firstTokenSeconds": 9.408,
      "firstToolSeconds": 13.327,
      "firstArtifactSeconds": 43.33,
      "decodeTokensPerSecond": 85.05,
      "prefillTokensPerSecond": 1006.41,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 22422,
      "outputTokensReported": 10930,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 40905,
      "apiOutputTokensReported": 5655,
      "peakSubtreeRssGiB": 20.6484375,
      "peakSystemUsedGiB": 92.95703125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-productivity-plan-and-estimate",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-qwen3-8-27b-q4-2026-11-03T11-05-02-154Z-szaj",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 90.058,
      "firstTokenSeconds": 17.517,
      "firstToolSeconds": 19.482,
      "firstArtifactSeconds": 85.856,
      "decodeTokensPerSecond": 23.55,
      "prefillTokensPerSecond": 437.16,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 4749,
      "outputTokensReported": 1581,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 35.8271484375,
      "peakSystemUsedGiB": 111.279296875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-productivity-plan-and-estimate",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-11-03T11-07-01-547Z-fdf1",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 37.761,
      "firstTokenSeconds": 12.142,
      "firstToolSeconds": 24.972,
      "firstArtifactSeconds": 35.191,
      "decodeTokensPerSecond": 57,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1222,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 6183,
      "outputTokensReported": 857,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 6180,
      "apiOutputTokensReported": 36,
      "peakSubtreeRssGiB": 9.9453125,
      "peakSystemUsedGiB": 91.41015625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 3,
      "completedSpecRequestGeneratedTokens": 1261,
      "specAcceptedTokensPerRound": {
        "n": 3,
        "median": 2.55,
        "mean": 2.5833333333333335,
        "min": 2.33,
        "max": 2.87
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-productivity-plan-and-estimate",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-11-03T11-08-12-542Z-k9ef",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 32.184,
      "firstTokenSeconds": 9.335,
      "firstToolSeconds": 13.459,
      "firstArtifactSeconds": 21.518,
      "decodeTokensPerSecond": 75.04,
      "prefillTokensPerSecond": 1101.83,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 6169,
      "outputTokensReported": 911,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 6166,
      "apiOutputTokensReported": 42,
      "peakSubtreeRssGiB": 14.1494140625,
      "peakSystemUsedGiB": 86.431640625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-mlx-productivity-plan-and-estimate",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "plan-and-estimate",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-03T11-09-21-661Z-7ly7",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 57.565,
      "firstTokenSeconds": 12.236,
      "firstToolSeconds": 13.062,
      "firstArtifactSeconds": 51.413,
      "decodeTokensPerSecond": 40,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 943,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 4817,
      "outputTokensReported": 1367,
      "tokenSource": "engine-log",
      "apiInputTokensReported": null,
      "apiOutputTokensReported": null,
      "peakSubtreeRssGiB": 17.1474609375,
      "peakSystemUsedGiB": 107.328125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 2,
      "completedSpecRequestGeneratedTokens": 1509,
      "specAcceptedTokensPerRound": {
        "n": 2,
        "median": 2.37,
        "mean": 2.37,
        "min": 2.03,
        "max": 2.71
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-productivity-meeting-followup",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "meeting-followup",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-11-03T11-10-56-975Z-gv9z",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 218.969,
      "firstTokenSeconds": 11.794,
      "firstToolSeconds": 18.005,
      "firstArtifactSeconds": 23.529,
      "decodeTokensPerSecond": 68,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1615.5,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 4,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 50958,
      "outputTokensReported": 4077,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 18770,
      "apiOutputTokensReported": 376,
      "peakSubtreeRssGiB": 10.0439453125,
      "peakSystemUsedGiB": 91.27734375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 12,
      "completedSpecRequestGeneratedTokens": 5429,
      "specAcceptedTokensPerRound": {
        "n": 12,
        "median": 3.02,
        "mean": 3.1425,
        "min": 2.81,
        "max": 3.8
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "decisionClarity": 5,
        "grounding": 6,
        "actionability": 4,
        "brevity": 6
      }
    },
    {
      "id": "r1-gemma-llamacpp-productivity-meeting-followup",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "meeting-followup",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-meeting-followup/meeting-followup/meeting-followup-gemma4-12b-q4-2026-11-03T11-15-09-522Z-1hbv",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 175.344,
      "firstTokenSeconds": 10.862,
      "firstToolSeconds": 15.087,
      "firstArtifactSeconds": 22.154,
      "decodeTokensPerSecond": 105.02,
      "prefillTokensPerSecond": 1266.07,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 23346,
      "outputTokensReported": 4266,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 32810,
      "apiOutputTokensReported": 2504,
      "peakSubtreeRssGiB": 18.783203125,
      "peakSystemUsedGiB": 90.658203125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "decisionClarity": 6,
        "grounding": 7,
        "actionability": 6,
        "brevity": 7
      }
    },
    {
      "id": "r1-qwen-mlx-productivity-meeting-followup",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "meeting-followup",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-11-03T11-18-38-160Z-0qki",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 133.67,
      "firstTokenSeconds": 16.44,
      "firstToolSeconds": 18.128,
      "firstArtifactSeconds": 32.452,
      "decodeTokensPerSecond": 49,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 857,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 2,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 12675,
      "outputTokensReported": 492,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 2882,
      "apiOutputTokensReported": 497,
      "peakSubtreeRssGiB": 17.263671875,
      "peakSystemUsedGiB": 108.8505859375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 3,
      "completedSpecRequestGeneratedTokens": 751,
      "specAcceptedTokensPerRound": {
        "n": 3,
        "median": 2.8,
        "mean": 2.75,
        "min": 2.62,
        "max": 2.83
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "decisionClarity": 7,
        "grounding": 6,
        "actionability": 7,
        "brevity": 7
      }
    },
    {
      "id": "r1-qwen-llamacpp-productivity-meeting-followup",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "meeting-followup",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-meeting-followup/meeting-followup/meeting-followup-qwen3-8-27b-q4-2026-11-03T11-21-25-208Z-9j9y",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 141.405,
      "firstTokenSeconds": 19.497,
      "firstToolSeconds": 21.079,
      "firstArtifactSeconds": 36.395,
      "decodeTokensPerSecond": 53.75,
      "prefillTokensPerSecond": 612.3,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 12182,
      "outputTokensReported": 954,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 2821,
      "apiOutputTokensReported": 480,
      "peakSubtreeRssGiB": 37.3388671875,
      "peakSystemUsedGiB": 111.703125,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "decisionClarity": 7,
        "grounding": 6,
        "actionability": 7,
        "brevity": 7
      }
    },
    {
      "id": "r1-gemma-llamacpp-productivity-craftbook-spreadsheet-model",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "craftbook-spreadsheet-model",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-11-03T11-24-20-468Z-rbkm",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 154.238,
      "firstTokenSeconds": 15.358,
      "firstToolSeconds": 25.791,
      "firstArtifactSeconds": 25.792,
      "decodeTokensPerSecond": 94.04,
      "prefillTokensPerSecond": 1277.21,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 50713,
      "outputTokensReported": 6544,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 43927,
      "apiOutputTokensReported": 1433,
      "peakSubtreeRssGiB": 22.4892578125,
      "peakSystemUsedGiB": 103.6494140625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "workflow fidelity": 5,
        "usefulness": 4
      }
    },
    {
      "id": "r1-qwen-mlx-productivity-craftbook-spreadsheet-model",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "craftbook-spreadsheet-model",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-11-03T11-27-24-639Z-38e1",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9,
      "durationSeconds": 1263.132,
      "firstTokenSeconds": 25.596,
      "firstToolSeconds": 28.474,
      "firstArtifactSeconds": 89.848,
      "decodeTokensPerSecond": 25,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 566,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 4,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 48317,
      "outputTokensReported": 27954,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 40088,
      "apiOutputTokensReported": 3253,
      "peakSubtreeRssGiB": 17.583984375,
      "peakSystemUsedGiB": 107.818359375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 16,
      "completedSpecRequestGeneratedTokens": 29029,
      "specAcceptedTokensPerRound": {
        "n": 16,
        "median": 2.3099999999999996,
        "mean": 2.376875,
        "min": 2,
        "max": 2.89
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "workflow fidelity": 6,
        "usefulness": 3
      }
    },
    {
      "id": "r1-qwen-llamacpp-productivity-craftbook-spreadsheet-model",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "craftbook-spreadsheet-model",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-qwen3-8-27b-q4-2026-11-03T11-48-59-464Z-rdus",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 565.188,
      "firstTokenSeconds": 30.55,
      "firstToolSeconds": 34.676,
      "firstArtifactSeconds": 84.947,
      "decodeTokensPerSecond": 29.18,
      "prefillTokensPerSecond": 401.65,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 47229,
      "outputTokensReported": 11777,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 47189,
      "apiOutputTokensReported": 2456,
      "peakSubtreeRssGiB": 43.193359375,
      "peakSystemUsedGiB": 122.3037109375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "workflow fidelity": 6,
        "usefulness": 5
      }
    },
    {
      "id": "r1-gemma-mlx-productivity-craftbook-spreadsheet-model",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "craftbook-spreadsheet-model",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-11-03T11-59-00-041Z-xt4q",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 376.645,
      "firstTokenSeconds": 20.607,
      "firstToolSeconds": 37.21,
      "firstArtifactSeconds": 37.211,
      "decodeTokensPerSecond": 53,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1487,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 5,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 64126,
      "outputTokensReported": 13110,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 40417,
      "apiOutputTokensReported": 2962,
      "peakSubtreeRssGiB": 10.814453125,
      "peakSystemUsedGiB": 92.4462890625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 12,
      "completedSpecRequestGeneratedTokens": 14370,
      "specAcceptedTokensPerRound": {
        "n": 12,
        "median": 2.925,
        "mean": 3.1108333333333333,
        "min": 2.66,
        "max": 3.9
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "workflow fidelity": 5,
        "usefulness": 4
      }
    },
    {
      "id": "r1-qwen-mlx-productivity-conflict-synthesis",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-03T12-05-50-901Z-b4hg",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": false,
      "failureClass": "model",
      "fixedComposite": 5.5,
      "durationSeconds": 1117.971,
      "firstTokenSeconds": 27.833,
      "firstToolSeconds": 33.652,
      "firstArtifactSeconds": 351.972,
      "decodeTokensPerSecond": 27,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 522,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 6,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 141201,
      "outputTokensReported": 20602,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 69217,
      "apiOutputTokensReported": 3314,
      "peakSubtreeRssGiB": 17.2919921875,
      "peakSystemUsedGiB": 108.1201171875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 31,
      "completedSpecRequestGeneratedTokens": 22765,
      "specAcceptedTokensPerRound": {
        "n": 31,
        "median": 2.74,
        "mean": 2.6503225806451614,
        "min": 2.02,
        "max": 3
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-productivity-conflict-synthesis",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-qwen3-8-27b-q4-2026-11-03T12-25-02-949Z-f67h",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9,
      "durationSeconds": 336.262,
      "firstTokenSeconds": 32.783,
      "firstToolSeconds": 36.045,
      "firstArtifactSeconds": 150.78,
      "decodeTokensPerSecond": 33.14,
      "prefillTokensPerSecond": 416.69,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 50047,
      "outputTokensReported": 6358,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 30657,
      "apiOutputTokensReported": 1055,
      "peakSubtreeRssGiB": 44.8349609375,
      "peakSystemUsedGiB": 118.1435546875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-mlx-productivity-conflict-synthesis",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-11-03T12-31-12-895Z-84z4",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 37.848,
      "firstTokenSeconds": 17.099,
      "firstToolSeconds": 23.108,
      "firstArtifactSeconds": 33.812,
      "decodeTokensPerSecond": 51,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1083,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 11150,
      "outputTokensReported": 503,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 11147,
      "apiOutputTokensReported": 68,
      "peakSubtreeRssGiB": 9.955078125,
      "peakSystemUsedGiB": 92.921875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 3,
      "completedSpecRequestGeneratedTokens": 851,
      "specAcceptedTokensPerRound": {
        "n": 3,
        "median": 2.83,
        "mean": 2.8000000000000003,
        "min": 2.6,
        "max": 2.97
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-gemma-llamacpp-productivity-conflict-synthesis",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "conflict-synthesis",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-11-03T12-32-19-113Z-sup8",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 32.257,
      "firstTokenSeconds": 13.327,
      "firstToolSeconds": 16.799,
      "firstArtifactSeconds": 22.87,
      "decodeTokensPerSecond": 88.45,
      "prefillTokensPerSecond": 1161.98,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 11173,
      "outputTokensReported": 855,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 11170,
      "apiOutputTokensReported": 61,
      "peakSubtreeRssGiB": 14.1396484375,
      "peakSystemUsedGiB": 87.8310546875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "notApplicable",
      "advisoryAxes": null
    },
    {
      "id": "r1-qwen-llamacpp-productivity-docblocks-theme-roundtrip",
      "provider": "llama-cpp",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-llamacpp",
      "suite": "productivity",
      "scenario": "docblocks-theme-roundtrip",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-llamacpp-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-qwen3-8-27b-q4-2026-11-03T12-33-22-452Z-3abv",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 172.209,
      "firstTokenSeconds": 47.83,
      "firstToolSeconds": 48.715,
      "firstArtifactSeconds": 141.284,
      "decodeTokensPerSecond": 33.73,
      "prefillTokensPerSecond": 489.49,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 35303,
      "outputTokensReported": 2674,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 27961,
      "apiOutputTokensReported": 505,
      "peakSubtreeRssGiB": 42.8935546875,
      "peakSystemUsedGiB": 116.1162109375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "grounding": 4,
        "candor": 4,
        "usefulness": 4
      }
    },
    {
      "id": "r1-gemma-mlx-productivity-docblocks-theme-roundtrip",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "docblocks-theme-roundtrip",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-11-03T12-36-49-628Z-sdya",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 349.351,
      "firstTokenSeconds": 16.969,
      "firstToolSeconds": 21.252,
      "firstArtifactSeconds": 343.835,
      "decodeTokensPerSecond": 61,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1191,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 3,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 71458,
      "outputTokensReported": 10661,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 46987,
      "apiOutputTokensReported": 8382,
      "peakSubtreeRssGiB": 10.1904296875,
      "peakSystemUsedGiB": 91.8359375,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 13,
      "completedSpecRequestGeneratedTokens": 11843,
      "specAcceptedTokensPerRound": {
        "n": 13,
        "median": 3.21,
        "mean": 3.2346153846153847,
        "min": 1.67,
        "max": 4
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "grounding": 6,
        "candor": 6,
        "usefulness": 5
      }
    },
    {
      "id": "r1-gemma-llamacpp-productivity-docblocks-theme-roundtrip",
      "provider": "llama-cpp",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-llamacpp",
      "suite": "productivity",
      "scenario": "docblocks-theme-roundtrip",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-llamacpp-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-11-03T12-43-12-327Z-0sh7",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 10,
      "durationSeconds": 122.175,
      "firstTokenSeconds": 16.716,
      "firstToolSeconds": 19.748,
      "firstArtifactSeconds": 106.823,
      "decodeTokensPerSecond": 83.19,
      "prefillTokensPerSecond": 931.24,
      "observedMlxPrefillTokensPerSecond": null,
      "observedMlxPrefillSource": null,
      "observedMlxPrefillSampleCount": null,
      "observedMlxPrefillAggregation": null,
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "sum request decoded tokens / sum request decode seconds",
      "inputTokensReported": 25525,
      "outputTokensReported": 1692,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 30862,
      "apiOutputTokensReported": 227,
      "peakSubtreeRssGiB": 24.556640625,
      "peakSystemUsedGiB": 96.482421875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": null,
      "completedSpecRequestGeneratedTokens": null,
      "specAcceptedTokensPerRound": {
        "n": 0,
        "median": null,
        "mean": null,
        "min": null,
        "max": null
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "grounding": 5,
        "candor": 5,
        "usefulness": 2
      }
    },
    {
      "id": "r1-qwen-mlx-productivity-docblocks-theme-roundtrip",
      "provider": "mlx",
      "model": "qwen3.8-27b-q4",
      "targetId": "qwen-mlx",
      "suite": "productivity",
      "scenario": "docblocks-theme-roundtrip",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-11-03T12-45-48-219Z-1o2j",
      "comparisonIncluded": true,
      "comparisonIssue": null,
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 9.5,
      "durationSeconds": 180.042,
      "firstTokenSeconds": 37.698,
      "firstToolSeconds": 39.997,
      "firstArtifactSeconds": 149.449,
      "decodeTokensPerSecond": 28,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 743,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 1,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 42333,
      "outputTokensReported": 1784,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 27141,
      "apiOutputTokensReported": 478,
      "peakSubtreeRssGiB": 17.3779296875,
      "peakSystemUsedGiB": 109.451171875,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 11,
      "completedSpecRequestGeneratedTokens": 2876,
      "specAcceptedTokensPerRound": {
        "n": 11,
        "median": 2.68,
        "mean": 2.5454545454545454,
        "min": 1.9,
        "max": 2.98
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "grounding": 2,
        "candor": 6,
        "usefulness": 6
      }
    },
    {
      "id": "r1-gemma-mlx-productivity-craftbook-powerpoint-deck",
      "provider": "mlx",
      "model": "gemma4-12b-q4",
      "targetId": "gemma-mlx",
      "suite": "productivity",
      "scenario": "craftbook-powerpoint-deck",
      "repetition": 1,
      "runDir": "evals/runs/scorecard-2026-11-03-m5max-fixed-paired/r1-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-11-03T12-49-23-654Z-l0ag",
      "comparisonIncluded": false,
      "comparisonIssue": "Frozen comparison changed: fixtures: \"bin/ab-strategy.ts\", \"strategy/books.ts\", \"strategy/engine-metrics.ts\", \"strategy/summary.ts\", \"types.ts\" (see identity-drift-after-trial.json)",
      "success": true,
      "failureClass": "pass",
      "fixedComposite": 8.5,
      "durationSeconds": 461.119,
      "firstTokenSeconds": 22.697,
      "firstToolSeconds": 33.462,
      "firstArtifactSeconds": 53.067,
      "decodeTokensPerSecond": 50,
      "prefillTokensPerSecond": null,
      "observedMlxPrefillTokensPerSecond": 1124,
      "observedMlxPrefillSource": "first-token-detail",
      "observedMlxPrefillSampleCount": 5,
      "observedMlxPrefillAggregation": "median-observed-events",
      "pureMlxPrefillDurationSeconds": null,
      "decodeMethod": "median active 5-second pulse rates",
      "inputTokensReported": 112835,
      "outputTokensReported": 5579,
      "tokenSource": "engine-log",
      "apiInputTokensReported": 74415,
      "apiOutputTokensReported": 1164,
      "peakSubtreeRssGiB": 11.5361328125,
      "peakSystemUsedGiB": 117.7900390625,
      "gpuUtilization": null,
      "gpuUnavailable": true,
      "completedSpecRequestCount": 39,
      "completedSpecRequestGeneratedTokens": 8250,
      "specAcceptedTokensPerRound": {
        "n": 39,
        "median": 3.33,
        "mean": 3.2756410256410255,
        "min": 2.47,
        "max": 4
      },
      "speculationEvidence": true,
      "judgeStatus": "judged",
      "advisoryAxes": {
        "source-acquisition": 6,
        "outline-parity": 7,
        "grounding": 4,
        "convertibility": 6
      }
    }
  ]
}