{
  "schemaVersion": 1,
  "publicationId": "2026-10-27-m5max-native-0-1-54",
  "recoveredOn": "2026-11-03",
  "campaignSha256": "a64051dadd9e07e96cee37afd4314febcfc8f43c069701ab0e3fa5cb1cc98dc1",
  "parserSha256": "0e141545ec2555c7f927da1498cffe8423ae0970a3533bf5ac17b65436d1b4f3",
  "method": "Median of observed rounded first-token prefill rates within each trial, then median across trials. Partial provider-turn coverage, not duration-weighted aggregate throughput or pure prefill time. All attempts are represented, including failures and the original infrastructure exclusion.",
  "caveats": [
    "Internal tool-loop requests, cancelled turns and event-stream gaps may be absent. Coverage of every request is not asserted.",
    "Only exact recorded duplicates can be detected; reconnect replay without stable event IDs cannot be identified.",
    "Time to first token includes other work and cannot substitute for prefill duration. Rounded rates and trial token totals cannot reconstruct exact durations.",
    "These observations must not be treated as the same estimator as llama.cpp aggregate prompt tokens divided by prompt-evaluation seconds."
  ],
  "coverage": {
    "recordedMlxTrials": 258,
    "recoveredMlxTrials": 258,
    "observations": 2175,
    "structuredObservations": 0,
    "malformedLines": 0,
    "exactDuplicates": 0,
    "capabilityExcludedTrials": 1
  },
  "groups": [
    {
      "targetId": "qwen-mlx",
      "model": "qwen3.8-27b-q4",
      "suite": "all",
      "trials": 129,
      "observations": 793,
      "observedPrefillTokensPerSec": {
        "samples": 129,
        "median": 547,
        "min": 155,
        "max": 862.5
      }
    },
    {
      "targetId": "qwen-mlx",
      "model": "qwen3.8-27b-q4",
      "suite": "core",
      "trials": 33,
      "observations": 84,
      "observedPrefillTokensPerSec": {
        "samples": 33,
        "median": 564,
        "min": 397,
        "max": 805.5
      }
    },
    {
      "targetId": "qwen-mlx",
      "model": "qwen3.8-27b-q4",
      "suite": "productivity",
      "trials": 39,
      "observations": 213,
      "observedPrefillTokensPerSec": {
        "samples": 39,
        "median": 574,
        "min": 494,
        "max": 862.5
      }
    },
    {
      "targetId": "qwen-mlx",
      "model": "qwen3.8-27b-q4",
      "suite": "developer",
      "trials": 30,
      "observations": 252,
      "observedPrefillTokensPerSec": {
        "samples": 30,
        "median": 512,
        "min": 155,
        "max": 735.5
      }
    },
    {
      "targetId": "qwen-mlx",
      "model": "qwen3.8-27b-q4",
      "suite": "complex-work",
      "trials": 27,
      "observations": 244,
      "observedPrefillTokensPerSec": {
        "samples": 27,
        "median": 495,
        "min": 259,
        "max": 807
      }
    },
    {
      "targetId": "gemma-mlx",
      "model": "gemma4-12b-q4",
      "suite": "all",
      "trials": 129,
      "observations": 1382,
      "observedPrefillTokensPerSec": {
        "samples": 129,
        "median": 1084,
        "min": 231,
        "max": 1921
      }
    },
    {
      "targetId": "gemma-mlx",
      "model": "gemma4-12b-q4",
      "suite": "core",
      "trials": 33,
      "observations": 194,
      "observedPrefillTokensPerSec": {
        "samples": 33,
        "median": 1146,
        "min": 603,
        "max": 1463.5
      }
    },
    {
      "targetId": "gemma-mlx",
      "model": "gemma4-12b-q4",
      "suite": "productivity",
      "trials": 39,
      "observations": 415,
      "observedPrefillTokensPerSec": {
        "samples": 39,
        "median": 1146,
        "min": 893,
        "max": 1921
      }
    },
    {
      "targetId": "gemma-mlx",
      "model": "gemma4-12b-q4",
      "suite": "developer",
      "trials": 30,
      "observations": 360,
      "observedPrefillTokensPerSec": {
        "samples": 30,
        "median": 1052.5,
        "min": 231,
        "max": 1534.5
      }
    },
    {
      "targetId": "gemma-mlx",
      "model": "gemma4-12b-q4",
      "suite": "complex-work",
      "trials": 27,
      "observations": 413,
      "observedPrefillTokensPerSec": {
        "samples": 27,
        "median": 712,
        "min": 461.5,
        "max": 1058
      }
    }
  ],
  "trials": [
    {
      "cell": {
        "id": "r1-qwen-mlx-core-tictactoe",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "tictactoe",
        "repetition": 1
      },
      "trialId": "tictactoe-mlx-qwen3-8-27b-q4-2026-10-28T00-10-08-465Z-d99v",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-10-28T00-10-08-465Z-d99v/recording/chat-events.jsonl",
      "recordingSha256": "63d3cf8a702f85a30c64bf8a342f5bf13fae438e9f323da9313236a594e50d91",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-10-28T00-10-08-465Z-d99v/metrics.json",
      "metricsSha256": "f53d13a973296461bdbaad926fc49e97f1ef1c362df1be74f97d60fe28dbbcba",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 692.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T00:10:31.333Z",
            "sessionId": "9a1d18e0-0934-4658-995b-8ec340a7f58d",
            "source": "first-token-detail",
            "tokensPerSec": 767,
            "ttftMs": 20967
          },
          {
            "line": 225,
            "rx": "2026-10-28T00:11:07.191Z",
            "sessionId": "33ced9e3-0aa7-45de-850c-c607a32eccbe",
            "source": "first-token-detail",
            "tokensPerSec": 618,
            "ttftMs": 7648
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-tictactoe",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "tictactoe",
        "repetition": 1
      },
      "trialId": "tictactoe-mlx-gemma4-12b-q4-2026-10-28T00-16-25-025Z-m3bu",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-10-28T00-16-25-025Z-m3bu/recording/chat-events.jsonl",
      "recordingSha256": "5e4149046be06ac76abb6b8fdd0c9617e8d1118159ac6841f7d8d656a17c0d33",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-10-28T00-16-25-025Z-m3bu/metrics.json",
      "metricsSha256": "89b3b5e4d738ee08825d4b9500c9cb287110faa5e8a0ce079cf517531f69e16a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1447,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 23,
            "rx": "2026-10-28T00:16:39.195Z",
            "sessionId": "acd51ce8-0405-47dd-943c-05515b988fbe",
            "source": "first-token-detail",
            "tokensPerSec": 1447,
            "ttftMs": 12271
          },
          {
            "line": 195,
            "rx": "2026-10-28T00:17:16.097Z",
            "sessionId": "1d2c3da6-88de-44ce-b424-bce329d8cbf6",
            "source": "first-token-detail",
            "tokensPerSec": 1572,
            "ttftMs": 4239
          },
          {
            "line": 225,
            "rx": "2026-10-28T00:17:30.079Z",
            "sessionId": "1d2c3da6-88de-44ce-b424-bce329d8cbf6",
            "source": "first-token-detail",
            "tokensPerSec": 1063,
            "ttftMs": 9288
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-petshop",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "petshop",
        "repetition": 1
      },
      "trialId": "petshop-mlx-gemma4-12b-q4-2026-10-28T00-30-51-467Z-02o4",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-10-28T00-30-51-467Z-02o4/recording/chat-events.jsonl",
      "recordingSha256": "2cb6c93b3ac662df1f7174fcdcbe02db590c9114b7264e32c03510ceef8a4f0c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-10-28T00-30-51-467Z-02o4/metrics.json",
      "metricsSha256": "4e11b7e4ee0d80e7b6da4e2cbfe1eb4892f7a2da67c2cb11ad3d134a16dbab03",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 853,
        "sampleCount": 18,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 18,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 23,
            "rx": "2026-10-28T00:31:08.243Z",
            "sessionId": "195df2be-2320-4687-8ac4-641ddeacfba4",
            "source": "first-token-detail",
            "tokensPerSec": 1148,
            "ttftMs": 15092
          },
          {
            "line": 141,
            "rx": "2026-10-28T00:31:38.295Z",
            "sessionId": "1c73b749-859d-4566-8a9f-607dea398636",
            "source": "first-token-detail",
            "tokensPerSec": 1474,
            "ttftMs": 10319
          },
          {
            "line": 256,
            "rx": "2026-10-28T00:31:59.132Z",
            "sessionId": "f5b17da4-4130-4a94-a320-dc84960ce553",
            "source": "first-token-detail",
            "tokensPerSec": 1182,
            "ttftMs": 4225
          },
          {
            "line": 298,
            "rx": "2026-10-28T00:32:19.482Z",
            "sessionId": "1c73b749-859d-4566-8a9f-607dea398636",
            "source": "first-token-detail",
            "tokensPerSec": 1144,
            "ttftMs": 11482
          },
          {
            "line": 1657,
            "rx": "2026-10-28T00:36:21.315Z",
            "sessionId": "41e4c202-5668-4d3a-801f-b23c11dd07b3",
            "source": "first-token-detail",
            "tokensPerSec": 966,
            "ttftMs": 11125
          },
          {
            "line": 2145,
            "rx": "2026-10-28T00:37:43.859Z",
            "sessionId": "e380f6f9-4395-4e10-b8fe-988343e88884",
            "source": "first-token-detail",
            "tokensPerSec": 826,
            "ttftMs": 14195
          },
          {
            "line": 2536,
            "rx": "2026-10-28T00:38:47.037Z",
            "sessionId": "c54da5db-fa7e-4da5-8e2a-1b7f7963772f",
            "source": "first-token-detail",
            "tokensPerSec": 933,
            "ttftMs": 5704
          },
          {
            "line": 2588,
            "rx": "2026-10-28T00:38:58.038Z",
            "sessionId": "1c73b749-859d-4566-8a9f-607dea398636",
            "source": "first-token-detail",
            "tokensPerSec": 326,
            "ttftMs": 465
          },
          {
            "line": 2755,
            "rx": "2026-10-28T00:39:23.539Z",
            "sessionId": "c54da5db-fa7e-4da5-8e2a-1b7f7963772f",
            "source": "first-token-detail",
            "tokensPerSec": 632,
            "ttftMs": 1060
          },
          {
            "line": 2809,
            "rx": "2026-10-28T00:39:34.808Z",
            "sessionId": "e380f6f9-4395-4e10-b8fe-988343e88884",
            "source": "first-token-detail",
            "tokensPerSec": 462,
            "ttftMs": 718
          },
          {
            "line": 4485,
            "rx": "2026-10-28T00:44:00.576Z",
            "sessionId": "41e4c202-5668-4d3a-801f-b23c11dd07b3",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 1034
          },
          {
            "line": 4889,
            "rx": "2026-10-28T00:45:00.860Z",
            "sessionId": "41e4c202-5668-4d3a-801f-b23c11dd07b3",
            "source": "first-token-detail",
            "tokensPerSec": 880,
            "ttftMs": 3574
          },
          {
            "line": 5372,
            "rx": "2026-10-28T00:46:06.450Z",
            "sessionId": "3e4dc15e-1aa2-4530-84f2-08232fb6f4b7",
            "source": "first-token-detail",
            "tokensPerSec": 54,
            "ttftMs": 69334
          },
          {
            "line": 5515,
            "rx": "2026-10-28T00:46:43.207Z",
            "sessionId": "c54da5db-fa7e-4da5-8e2a-1b7f7963772f",
            "source": "first-token-detail",
            "tokensPerSec": 624,
            "ttftMs": 1090
          },
          {
            "line": 5742,
            "rx": "2026-10-28T00:47:38.681Z",
            "sessionId": "e380f6f9-4395-4e10-b8fe-988343e88884",
            "source": "first-token-detail",
            "tokensPerSec": 325,
            "ttftMs": 459
          },
          {
            "line": 5915,
            "rx": "2026-10-28T00:48:05.058Z",
            "sessionId": "77c8622d-deb4-493b-84b7-d00ceb309a5a",
            "source": "first-token-detail",
            "tokensPerSec": 966,
            "ttftMs": 4906
          },
          {
            "line": 5964,
            "rx": "2026-10-28T00:48:16.899Z",
            "sessionId": "1c73b749-859d-4566-8a9f-607dea398636",
            "source": "first-token-detail",
            "tokensPerSec": 447,
            "ttftMs": 715
          },
          {
            "line": 6122,
            "rx": "2026-10-28T00:48:42.353Z",
            "sessionId": "3e4dc15e-1aa2-4530-84f2-08232fb6f4b7",
            "source": "first-token-detail",
            "tokensPerSec": 1005,
            "ttftMs": 6563
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-petshop",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "petshop",
        "repetition": 1
      },
      "trialId": "petshop-mlx-qwen3-8-27b-q4-2026-10-28T00-59-53-840Z-tkdx",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-10-28T00-59-53-840Z-tkdx/recording/chat-events.jsonl",
      "recordingSha256": "75b5b5cee7ab200d9eb6f21eeab99abef3506aaf4fb3df7fd0d958d9ad01acb2",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-10-28T00-59-53-840Z-tkdx/metrics.json",
      "metricsSha256": "b2cef4a673796966a0eeb249a8d06b92fd0f0f8851b4815b98947ef60cb2ff18",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 561,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T01:00:21.750Z",
            "sessionId": "33fbbde6-e3ba-4dde-85d4-ad3947390f4f",
            "source": "first-token-detail",
            "tokensPerSec": 585,
            "ttftMs": 26232
          },
          {
            "line": 88,
            "rx": "2026-10-28T01:00:50.759Z",
            "sessionId": "94e91282-64e7-4711-a37a-15c5f2fd6c60",
            "source": "first-token-detail",
            "tokensPerSec": 684,
            "ttftMs": 23176
          },
          {
            "line": 344,
            "rx": "2026-10-28T01:02:00.767Z",
            "sessionId": "5559d303-2dc7-45bb-b18b-c515ba999466",
            "source": "first-token-detail",
            "tokensPerSec": 561,
            "ttftMs": 35268
          },
          {
            "line": 1536,
            "rx": "2026-10-28T01:05:22.651Z",
            "sessionId": "5ec65775-2a44-41de-9294-96863b0a37fa",
            "source": "first-token-detail",
            "tokensPerSec": 474,
            "ttftMs": 10992
          },
          {
            "line": 3028,
            "rx": "2026-10-28T01:09:42.114Z",
            "sessionId": "7146cef9-0bad-448f-aab8-78213c2850e1",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 24160
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-tankcombat",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "tankcombat",
        "repetition": 1
      },
      "trialId": "tankcombat-mlx-gemma4-12b-q4-2026-10-28T01-27-56-400Z-0tfe",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-10-28T01-27-56-400Z-0tfe/recording/chat-events.jsonl",
      "recordingSha256": "d2ae0ae291aecc84bfad622da0a28f2e680929fea9258c4194fec988e89fab50",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-10-28T01-27-56-400Z-0tfe/metrics.json",
      "metricsSha256": "83c28e9b21729b7da03f7091a72566a07cbe47dc49e955e6d60c60da70d6a65c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1378,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 24,
            "rx": "2026-10-28T01:28:13.465Z",
            "sessionId": "1e6b10dc-7e82-4747-81de-5f9a137dcf29",
            "source": "first-token-detail",
            "tokensPerSec": 1132,
            "ttftMs": 15389
          },
          {
            "line": 189,
            "rx": "2026-10-28T01:28:46.780Z",
            "sessionId": "4cd8125e-10a9-4dfe-83d3-32e920b8f28f",
            "source": "first-token-detail",
            "tokensPerSec": 1624,
            "ttftMs": 3827
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-tankcombat",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "tankcombat",
        "repetition": 1
      },
      "trialId": "tankcombat-mlx-qwen3-8-27b-q4-2026-10-28T01-32-10-142Z-baus",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-10-28T01-32-10-142Z-baus/recording/chat-events.jsonl",
      "recordingSha256": "eb2a794579dd1922cc9da7538b565b648443dcf6621ad7c6debd5d3c77464736",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-10-28T01-32-10-142Z-baus/metrics.json",
      "metricsSha256": "50e836ea7f27e867b0336feeceb9af8b1fae2ca01641d033f108cea03e3884de",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 769.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T01:32:33.128Z",
            "sessionId": "f67ba35d-ab3b-467a-9aee-a6fe3eab5467",
            "source": "first-token-detail",
            "tokensPerSec": 817,
            "ttftMs": 21309
          },
          {
            "line": 295,
            "rx": "2026-10-28T01:33:17.734Z",
            "sessionId": "7b89cc14-0d0b-440e-b2e6-28e4732bafe1",
            "source": "first-token-detail",
            "tokensPerSec": 722,
            "ttftMs": 4123
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-schema-migration",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "schema-migration",
        "repetition": 1
      },
      "trialId": "schema-migration-mlx-qwen3-8-27b-q4-2026-10-28T01-46-59-305Z-hzvp",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-10-28T01-46-59-305Z-hzvp/recording/chat-events.jsonl",
      "recordingSha256": "bc1d6b8359b3be62a31bead954f8e109f3fe33161ab2e8bc58e8f48506ff7df7",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-10-28T01-46-59-305Z-hzvp/metrics.json",
      "metricsSha256": "858af0a711b8ab3b0449b43b6f930dd9f9e0bf7d32eae72ca62f8a61ee65add1",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 547,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-28T01:47:15.359Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 543,
            "ttftMs": 14398
          },
          {
            "line": 111,
            "rx": "2026-10-28T01:47:40.059Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 547,
            "ttftMs": 7446
          },
          {
            "line": 152,
            "rx": "2026-10-28T01:47:55.035Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 593,
            "ttftMs": 8869
          },
          {
            "line": 234,
            "rx": "2026-10-28T01:48:22.920Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 7111
          },
          {
            "line": 297,
            "rx": "2026-10-28T01:48:36.801Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 496,
            "ttftMs": 912
          },
          {
            "line": 503,
            "rx": "2026-10-28T01:49:25.092Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 559,
            "ttftMs": 8862
          },
          {
            "line": 625,
            "rx": "2026-10-28T01:50:06.929Z",
            "sessionId": "cd71fbde-6083-420d-9d41-7109b0281a9e",
            "source": "first-token-detail",
            "tokensPerSec": 535,
            "ttftMs": 15403
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-schema-migration",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "schema-migration",
        "repetition": 1
      },
      "trialId": "schema-migration-mlx-gemma4-12b-q4-2026-10-28T01-55-10-372Z-cx0a",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-10-28T01-55-10-372Z-cx0a/recording/chat-events.jsonl",
      "recordingSha256": "0a59f2ee26c8ba6e95a68cffb5e36072d3d4b7dd0c1243ad6987fbbb9eca1cd7",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-10-28T01-55-10-372Z-cx0a/metrics.json",
      "metricsSha256": "a510f0fd2753619fbaaf16ebd55c1cc72856e7195eddc22ca4ceba85ad26bf1d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1045,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T01:55:23.152Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1044,
            "ttftMs": 11147
          },
          {
            "line": 179,
            "rx": "2026-10-28T01:55:56.908Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1079,
            "ttftMs": 4959
          },
          {
            "line": 200,
            "rx": "2026-10-28T01:56:07.244Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1316,
            "ttftMs": 4971
          },
          {
            "line": 228,
            "rx": "2026-10-28T01:56:17.654Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1486,
            "ttftMs": 784
          },
          {
            "line": 255,
            "rx": "2026-10-28T01:56:34.267Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1049,
            "ttftMs": 7350
          },
          {
            "line": 290,
            "rx": "2026-10-28T01:56:54.833Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1106,
            "ttftMs": 7834
          },
          {
            "line": 345,
            "rx": "2026-10-28T01:57:23.911Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1045,
            "ttftMs": 10560
          },
          {
            "line": 580,
            "rx": "2026-10-28T01:58:06.239Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 484,
            "ttftMs": 356
          },
          {
            "line": 815,
            "rx": "2026-10-28T01:58:52.881Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 368,
            "ttftMs": 627
          },
          {
            "line": 1066,
            "rx": "2026-10-28T01:59:41.406Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 988,
            "ttftMs": 11148
          },
          {
            "line": 1089,
            "rx": "2026-10-28T01:59:58.563Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 1150,
            "ttftMs": 10538
          },
          {
            "line": 1182,
            "rx": "2026-10-28T02:00:47.484Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 971,
            "ttftMs": 14817
          },
          {
            "line": 1571,
            "rx": "2026-10-28T02:01:53.994Z",
            "sessionId": "481c9b43-d8a7-4b6d-a7ef-19f6616dce0f",
            "source": "first-token-detail",
            "tokensPerSec": 426,
            "ttftMs": 470
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-failing-tests-spec",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "failing-tests-spec",
        "repetition": 1
      },
      "trialId": "failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-28T02-02-17-622Z-ixp1",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-28T02-02-17-622Z-ixp1/recording/chat-events.jsonl",
      "recordingSha256": "c464b4f6ec623cbc20cfa237d63d2085b55334f5511b81e923c8e244ffce3ee3",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-28T02-02-17-622Z-ixp1/metrics.json",
      "metricsSha256": "ce3741d2900f69c267d801acb3d78e48fa9ce433cba1c05d1fff361b860e2bba",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 600,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T02:02:32.552Z",
            "sessionId": "195e3f5e-c164-415a-83fc-b0dcec18f7a9",
            "source": "first-token-detail",
            "tokensPerSec": 541,
            "ttftMs": 13242
          },
          {
            "line": 130,
            "rx": "2026-10-28T02:03:07.702Z",
            "sessionId": "195e3f5e-c164-415a-83fc-b0dcec18f7a9",
            "source": "first-token-detail",
            "tokensPerSec": 659,
            "ttftMs": 7855
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-failing-tests-spec",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "failing-tests-spec",
        "repetition": 1
      },
      "trialId": "failing-tests-spec-mlx-gemma4-12b-q4-2026-10-28T02-04-51-967Z-dgwy",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-10-28T02-04-51-967Z-dgwy/recording/chat-events.jsonl",
      "recordingSha256": "02134828eedeb5f2fcf842a6fcd292a037e15710903d09d700e275f7f9d6b449",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-10-28T02-04-51-967Z-dgwy/metrics.json",
      "metricsSha256": "5bd5a74efea39bd622c9bff5f25cf74e4b608c250380eb8026c53ec74e86dd83",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1300,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-10-28T02:05:03.437Z",
            "sessionId": "f8fc5d9b-3f77-4a34-aebd-947d76299af5",
            "source": "first-token-detail",
            "tokensPerSec": 1300,
            "ttftMs": 9823
          },
          {
            "line": 174,
            "rx": "2026-10-28T02:06:00.343Z",
            "sessionId": "f8fc5d9b-3f77-4a34-aebd-947d76299af5",
            "source": "first-token-detail",
            "tokensPerSec": 1364,
            "ttftMs": 6067
          },
          {
            "line": 217,
            "rx": "2026-10-28T02:06:27.649Z",
            "sessionId": "f8fc5d9b-3f77-4a34-aebd-947d76299af5",
            "source": "first-token-detail",
            "tokensPerSec": 1380,
            "ttftMs": 7838
          },
          {
            "line": 257,
            "rx": "2026-10-28T02:06:57.491Z",
            "sessionId": "f8fc5d9b-3f77-4a34-aebd-947d76299af5",
            "source": "first-token-detail",
            "tokensPerSec": 1188,
            "ttftMs": 12136
          },
          {
            "line": 375,
            "rx": "2026-10-28T02:07:57.745Z",
            "sessionId": "f8fc5d9b-3f77-4a34-aebd-947d76299af5",
            "source": "first-token-detail",
            "tokensPerSec": 1035,
            "ttftMs": 1109
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-symptom-debug",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "symptom-debug",
        "repetition": 1
      },
      "trialId": "symptom-debug-mlx-gemma4-12b-q4-2026-10-28T02-10-37-236Z-x3jd",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-10-28T02-10-37-236Z-x3jd/recording/chat-events.jsonl",
      "recordingSha256": "db449a72e2734930272e33df0be062358aa265c57d7d1cb8b3ccf34b354814d2",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-10-28T02-10-37-236Z-x3jd/metrics.json",
      "metricsSha256": "2259a829295246632f156d08e4ab9a328f487c66694783e2f39e12e3d7a5ce86",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1165.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T02:10:48.990Z",
            "sessionId": "0ceb3e40-0d38-40ec-98da-793eb2dd69e1",
            "source": "first-token-detail",
            "tokensPerSec": 1303,
            "ttftMs": 10121
          },
          {
            "line": 1397,
            "rx": "2026-10-28T02:14:38.435Z",
            "sessionId": "0ceb3e40-0d38-40ec-98da-793eb2dd69e1",
            "source": "first-token-detail",
            "tokensPerSec": 1028,
            "ttftMs": 15280
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-symptom-debug",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "symptom-debug",
        "repetition": 1
      },
      "trialId": "symptom-debug-mlx-qwen3-8-27b-q4-2026-10-28T02-17-58-089Z-s8hd",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-10-28T02-17-58-089Z-s8hd/recording/chat-events.jsonl",
      "recordingSha256": "73aafe9e361715f3b36183c404f637c1f3684a043fd1f5df14b076d321c2b9df",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-10-28T02-17-58-089Z-s8hd/metrics.json",
      "metricsSha256": "422f3f87d58d22be0eaa4e09c76ec36812b726b86b3fa7ab35393252354f508b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 564,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T02:18:13.580Z",
            "sessionId": "83b9e4ec-a383-41bf-8f6a-35319bb0e670",
            "source": "first-token-detail",
            "tokensPerSec": 564,
            "ttftMs": 13847
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-data-wrangle",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "data-wrangle",
        "repetition": 1
      },
      "trialId": "data-wrangle-mlx-gemma4-12b-q4-2026-10-28T02-18-56-828Z-7owd",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-10-28T02-18-56-828Z-7owd/recording/chat-events.jsonl",
      "recordingSha256": "7797725f4903e6839f62bf1fb1900428d8188123e8f494e95a91ba2e648cea5b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-10-28T02-18-56-828Z-7owd/metrics.json",
      "metricsSha256": "b5317dca6c2ba219c6c1854e84fe19f7e0c2e5a2dfd5058947fe17382c7f3e45",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1207.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T02:19:08.470Z",
            "sessionId": "b72df6bb-ca54-4d44-9678-9cf04f443e0c",
            "source": "first-token-detail",
            "tokensPerSec": 1229,
            "ttftMs": 10007
          },
          {
            "line": 586,
            "rx": "2026-10-28T02:21:34.513Z",
            "sessionId": "b72df6bb-ca54-4d44-9678-9cf04f443e0c",
            "source": "first-token-detail",
            "tokensPerSec": 1229,
            "ttftMs": 10304
          },
          {
            "line": 697,
            "rx": "2026-10-28T02:22:10.498Z",
            "sessionId": "b72df6bb-ca54-4d44-9678-9cf04f443e0c",
            "source": "first-token-detail",
            "tokensPerSec": 1186,
            "ttftMs": 1722
          },
          {
            "line": 734,
            "rx": "2026-10-28T02:22:51.437Z",
            "sessionId": "b72df6bb-ca54-4d44-9678-9cf04f443e0c",
            "source": "first-token-detail",
            "tokensPerSec": 1089,
            "ttftMs": 16966
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-data-wrangle",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "data-wrangle",
        "repetition": 1
      },
      "trialId": "data-wrangle-mlx-qwen3-8-27b-q4-2026-10-28T02-25-27-434Z-v934",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-10-28T02-25-27-434Z-v934/recording/chat-events.jsonl",
      "recordingSha256": "8a7e5d110e2dd2fd1d6d9ee389885ca1f6de12427bb0b55697db45dba4b9ea89",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-10-28T02-25-27-434Z-v934/metrics.json",
      "metricsSha256": "fea69f10b8fd559047038aeed6c4fd29f245a9ec0668f51f6d8eb8c985986530",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 598,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T02:25:43.023Z",
            "sessionId": "dcebcc77-1960-4073-a92f-e3bb00990606",
            "source": "first-token-detail",
            "tokensPerSec": 598,
            "ttftMs": 13927
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-incident-postmortem",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "incident-postmortem",
        "repetition": 1
      },
      "trialId": "incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-28T02-38-04-781Z-frfy",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-28T02-38-04-781Z-frfy/recording/chat-events.jsonl",
      "recordingSha256": "12ff852dd02058bb939712c3a5fb0074faa76e5152156409d054dbf35759b947",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-28T02-38-04-781Z-frfy/metrics.json",
      "metricsSha256": "8130b71da44e8df4e140bdd85a0ad1e83098a8438c247492545d9c918d980ccb",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 549.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T02:38:32.848Z",
            "sessionId": "26d5e9ed-46cf-4df1-9cfb-98d3b497970a",
            "source": "first-token-detail",
            "tokensPerSec": 583,
            "ttftMs": 26419
          },
          {
            "line": 4987,
            "rx": "2026-10-28T02:53:23.831Z",
            "sessionId": "26d5e9ed-46cf-4df1-9cfb-98d3b497970a",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 22424
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-incident-postmortem",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "incident-postmortem",
        "repetition": 1
      },
      "trialId": "incident-postmortem-mlx-gemma4-12b-q4-2026-10-28T03-09-10-894Z-r7ig",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-10-28T03-09-10-894Z-r7ig/recording/chat-events.jsonl",
      "recordingSha256": "d9bccf09382d9d4b9efffbb69b06b3f213c4d4be0f894b08a66b8568bb0b9012",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-10-28T03-09-10-894Z-r7ig/metrics.json",
      "metricsSha256": "48d2d22e635c6737877f8d1de672da5f12b7af0e99ff9606747967f18767be3b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1113,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-28T03:09:28.386Z",
            "sessionId": "e41d1ba9-8d19-4d9d-9c48-be5e98ec24c0",
            "source": "first-token-detail",
            "tokensPerSec": 1074,
            "ttftMs": 15863
          },
          {
            "line": 676,
            "rx": "2026-10-28T03:12:00.525Z",
            "sessionId": "e41d1ba9-8d19-4d9d-9c48-be5e98ec24c0",
            "source": "first-token-detail",
            "tokensPerSec": 532,
            "ttftMs": 380
          },
          {
            "line": 875,
            "rx": "2026-10-28T03:12:45.333Z",
            "sessionId": "e41d1ba9-8d19-4d9d-9c48-be5e98ec24c0",
            "source": "first-token-detail",
            "tokensPerSec": 1209,
            "ttftMs": 11868
          },
          {
            "line": 920,
            "rx": "2026-10-28T03:12:58.885Z",
            "sessionId": "e41d1ba9-8d19-4d9d-9c48-be5e98ec24c0",
            "source": "first-token-detail",
            "tokensPerSec": 1152,
            "ttftMs": 859
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-ops-runbook-anomaly",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "ops-runbook-anomaly",
        "repetition": 1
      },
      "trialId": "ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-28T03-13-38-684Z-ggrp",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-28T03-13-38-684Z-ggrp/recording/chat-events.jsonl",
      "recordingSha256": "1ffa4ca3ce7507d5ac033887fe0986de69e155c3eba2ac726bf8f0c02253f269",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-28T03-13-38-684Z-ggrp/metrics.json",
      "metricsSha256": "034f8dce896195bb5a82812516e314ce99a3a818d8a336cbde756f9ebf6c3fbb",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 570,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T03:14:06.276Z",
            "sessionId": "7826bb13-79e3-4d6c-a431-9e2cb46cd4a6",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 25936
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-ops-runbook-anomaly",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "ops-runbook-anomaly",
        "repetition": 1
      },
      "trialId": "ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-28T03-18-10-442Z-tjt4",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-28T03-18-10-442Z-tjt4/recording/chat-events.jsonl",
      "recordingSha256": "7783a47c13e8d371fab9040dc1500863e24055dbc45356029fddef1dd643567a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-28T03-18-10-442Z-tjt4/metrics.json",
      "metricsSha256": "877831481aada0f1a866d219a473115f612225262420662e1b0138bc22d50417",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1413,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-28T03:18:27.044Z",
            "sessionId": "1e1075fa-ea3f-4500-b226-1b1d81b6b2e8",
            "source": "first-token-detail",
            "tokensPerSec": 1160,
            "ttftMs": 14959
          },
          {
            "line": 161,
            "rx": "2026-10-28T03:18:56.675Z",
            "sessionId": "1e1075fa-ea3f-4500-b226-1b1d81b6b2e8",
            "source": "first-token-detail",
            "tokensPerSec": 1433,
            "ttftMs": 4383
          },
          {
            "line": 205,
            "rx": "2026-10-28T03:19:16.397Z",
            "sessionId": "1e1075fa-ea3f-4500-b226-1b1d81b6b2e8",
            "source": "first-token-detail",
            "tokensPerSec": 1413,
            "ttftMs": 4516
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-plan-and-estimate",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "plan-and-estimate",
        "repetition": 1
      },
      "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T03-23-29-007Z-psno",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T03-23-29-007Z-psno/recording/chat-events.jsonl",
      "recordingSha256": "219b848ef766a24d707a74b7893f3f8d5e57272cebc7900747c9f7044ac9e3d1",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T03-23-29-007Z-psno/metrics.json",
      "metricsSha256": "39c43ca9fab44183efbc54ca304ba80cf6817a87d98783be7a8d0cd2aa52c3a7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1201,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T03:23:41.325Z",
            "sessionId": "2cc7c9c2-e3f3-475f-820d-48df9458c64d",
            "source": "first-token-detail",
            "tokensPerSec": 1201,
            "ttftMs": 10689
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-plan-and-estimate",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "plan-and-estimate",
        "repetition": 1
      },
      "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T03-25-19-783Z-ky1v",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T03-25-19-783Z-ky1v/recording/chat-events.jsonl",
      "recordingSha256": "20fcb2f25c5be8d7db9f2bf1c9702d875f6382e9898f863994572bf04f5d4360",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T03-25-19-783Z-ky1v/metrics.json",
      "metricsSha256": "5150e5ebe6c282f09b14c6c1533613c8e27ebbf5082684e960db29e2c41bc1e0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 658,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T03:25:34.784Z",
            "sessionId": "90c8b8c9-fb8f-4ecb-96b8-f76a87dc234f",
            "source": "first-token-detail",
            "tokensPerSec": 658,
            "ttftMs": 13348
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-core-conflict-synthesis",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "conflict-synthesis",
        "repetition": 1
      },
      "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T03-26-49-015Z-byen",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T03-26-49-015Z-byen/recording/chat-events.jsonl",
      "recordingSha256": "22e0af94f9de9d28b5e1bc6aadf3b0a35703b3698fa321f07feeec8b447861ef",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T03-26-49-015Z-byen/metrics.json",
      "metricsSha256": "cc511940a559801709faec9af8a1222b5ee5d7f2fbb6cebd0ad56c837bcaaabc",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1460.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T03:27:04.459Z",
            "sessionId": "cf9ceb1c-debd-438a-912f-2166a742e120",
            "source": "first-token-detail",
            "tokensPerSec": 1334,
            "ttftMs": 13807
          },
          {
            "line": 152,
            "rx": "2026-10-28T03:27:34.581Z",
            "sessionId": "cf9ceb1c-debd-438a-912f-2166a742e120",
            "source": "first-token-detail",
            "tokensPerSec": 1587,
            "ttftMs": 3732
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-core-conflict-synthesis",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "conflict-synthesis",
        "repetition": 1
      },
      "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T03-31-52-036Z-1i5n",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T03-31-52-036Z-1i5n/recording/chat-events.jsonl",
      "recordingSha256": "05d5288f5f712a8682ae988a77bf7fddd063c26e3c97f531e9c7c474015fd555",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T03-31-52-036Z-1i5n/metrics.json",
      "metricsSha256": "f54bdc86922a49c40dc1423dbc5cb767b7f403613798d51c1a222e2923d5279f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 508,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-28T03:32:22.600Z",
            "sessionId": "859ec682-9798-49bb-8da4-872858a05f91",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 28909
          },
          {
            "line": 1591,
            "rx": "2026-10-28T03:37:12.039Z",
            "sessionId": "859ec682-9798-49bb-8da4-872858a05f91",
            "source": "first-token-detail",
            "tokensPerSec": 508,
            "ttftMs": 11815
          },
          {
            "line": 1850,
            "rx": "2026-10-28T03:38:16.738Z",
            "sessionId": "859ec682-9798-49bb-8da4-872858a05f91",
            "source": "first-token-detail",
            "tokensPerSec": 496,
            "ttftMs": 12156
          },
          {
            "line": 1932,
            "rx": "2026-10-28T03:38:53.710Z",
            "sessionId": "859ec682-9798-49bb-8da4-872858a05f91",
            "source": "first-token-detail",
            "tokensPerSec": 508,
            "ttftMs": 13964
          },
          {
            "line": 2017,
            "rx": "2026-10-28T03:39:35.295Z",
            "sessionId": "859ec682-9798-49bb-8da4-872858a05f91",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 15376
          },
          {
            "line": 2402,
            "rx": "2026-10-28T03:49:36.342Z",
            "sessionId": "859ec682-9798-49bb-8da4-872858a05f91",
            "source": "first-token-detail",
            "tokensPerSec": 828,
            "ttftMs": 21824
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-constrained-comms",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "constrained-comms",
        "repetition": 1
      },
      "trialId": "constrained-comms-mlx-qwen3-8-27b-q4-2026-10-28T04-02-37-132Z-iyzn",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-10-28T04-02-37-132Z-iyzn/recording/chat-events.jsonl",
      "recordingSha256": "460c59e3d0f56fbe320cdb82e67eb6c84a69fed6fa18cb82a032b70ada6d9ea2",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-10-28T04-02-37-132Z-iyzn/metrics.json",
      "metricsSha256": "1097d086e694036beab6f103b7757e90df53d11484673330449ec2e812c31893",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 693,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T04:03:03.059Z",
            "sessionId": "977e16a8-6435-43a3-ba12-b24da4526b9a",
            "source": "first-token-detail",
            "tokensPerSec": 645,
            "ttftMs": 24286
          },
          {
            "line": 85,
            "rx": "2026-10-28T04:03:20.493Z",
            "sessionId": "977e16a8-6435-43a3-ba12-b24da4526b9a",
            "source": "first-token-detail",
            "tokensPerSec": 716,
            "ttftMs": 6557
          },
          {
            "line": 133,
            "rx": "2026-10-28T04:03:37.888Z",
            "sessionId": "977e16a8-6435-43a3-ba12-b24da4526b9a",
            "source": "first-token-detail",
            "tokensPerSec": 706,
            "ttftMs": 4378
          },
          {
            "line": 170,
            "rx": "2026-10-28T04:03:55.706Z",
            "sessionId": "977e16a8-6435-43a3-ba12-b24da4526b9a",
            "source": "first-token-detail",
            "tokensPerSec": 680,
            "ttftMs": 7130
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-constrained-comms",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "constrained-comms",
        "repetition": 1
      },
      "trialId": "constrained-comms-mlx-gemma4-12b-q4-2026-10-28T04-06-22-235Z-ukrp",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-10-28T04-06-22-235Z-ukrp/recording/chat-events.jsonl",
      "recordingSha256": "0c5691b89e04d189f156a02129927ae064691b2d1e168f91412f85929640716b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-10-28T04-06-22-235Z-ukrp/metrics.json",
      "metricsSha256": "be8ad661256fa9a7a3fe500fd4e987b90f85bad4663f9fe558b4dff6ec4f31c9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1295,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T04:06:38.380Z",
            "sessionId": "bf603b67-862d-4460-8451-f5cdbd71489c",
            "source": "first-token-detail",
            "tokensPerSec": 1211,
            "ttftMs": 14516
          },
          {
            "line": 150,
            "rx": "2026-10-28T04:07:08.298Z",
            "sessionId": "bf603b67-862d-4460-8451-f5cdbd71489c",
            "source": "first-token-detail",
            "tokensPerSec": 1379,
            "ttftMs": 4246
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-records-intake",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "records-intake",
        "repetition": 1
      },
      "trialId": "records-intake-mlx-qwen3-8-27b-q4-2026-10-28T04-07-46-716Z-w948",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-10-28T04-07-46-716Z-w948/recording/chat-events.jsonl",
      "recordingSha256": "26e0be6e5d997fac3497984569072a58bab94f3b9ec2bf83afbea32c163b5f11",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-10-28T04-07-46-716Z-w948/metrics.json",
      "metricsSha256": "c77c77c1f8a1516c779026715fa8d384662889f41c6f10381ab45067660ad3f2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 598,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T04:08:02.541Z",
            "sessionId": "5d0cdbe0-4ffe-4d01-931d-010cce1f5c41",
            "source": "first-token-detail",
            "tokensPerSec": 598,
            "ttftMs": 14183
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-records-intake",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "records-intake",
        "repetition": 1
      },
      "trialId": "records-intake-mlx-gemma4-12b-q4-2026-10-28T04-12-08-219Z-qnj8",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-10-28T04-12-08-219Z-qnj8/recording/chat-events.jsonl",
      "recordingSha256": "2777f23ac27c9b40f76567693c30133149341cbc9e8602ea97d916373fe6d1a9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-10-28T04-12-08-219Z-qnj8/metrics.json",
      "metricsSha256": "5d2b3b8292ff245a39456804bc0529f38c6468cb62ceb2aeb6cd9ecd0dbc266a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1146,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T04:12:20.706Z",
            "sessionId": "ffd78ff8-2f02-4188-a91f-1d848bbb1629",
            "source": "first-token-detail",
            "tokensPerSec": 1113,
            "ttftMs": 10860
          },
          {
            "line": 381,
            "rx": "2026-10-28T04:13:42.797Z",
            "sessionId": "ffd78ff8-2f02-4188-a91f-1d848bbb1629",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 7479
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-plan-and-estimate",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "plan-and-estimate",
        "repetition": 1
      },
      "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T04-19-43-086Z-q6yd",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T04-19-43-086Z-q6yd/recording/chat-events.jsonl",
      "recordingSha256": "ae083ee1df97ebba4be8ed17b9dfbc60bccfc27f82c040d0696c7adab0baf952",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-28T04-19-43-086Z-q6yd/metrics.json",
      "metricsSha256": "e387af1f04b7ea38ebf8567926cb024285fe6ab281f167cff8a75d33a6bed243",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1075,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T04:19:55.932Z",
            "sessionId": "1d0d195d-a01c-454a-aec1-5bc8a138626e",
            "source": "first-token-detail",
            "tokensPerSec": 1075,
            "ttftMs": 11224
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-plan-and-estimate",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "plan-and-estimate",
        "repetition": 1
      },
      "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T04-21-39-794Z-ozts",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T04-21-39-794Z-ozts/recording/chat-events.jsonl",
      "recordingSha256": "8864b234cf1c352ecaa7dea7c0c17a2c66aed7aee724b34bb0b07d6184337874",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-28T04-21-39-794Z-ozts/metrics.json",
      "metricsSha256": "0ec828b576998b762b82fd45ea878b06ac386b4a9c2cdc329bfa56fcb5c0bf4e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 678,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T04:21:54.056Z",
            "sessionId": "1ca1fb07-059e-4e35-9355-06bbd4197979",
            "source": "first-token-detail",
            "tokensPerSec": 678,
            "ttftMs": 12620
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-meeting-followup",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "meeting-followup",
        "repetition": 1
      },
      "trialId": "meeting-followup-mlx-gemma4-12b-q4-2026-10-28T04-22-52-767Z-gfxj",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-10-28T04-22-52-767Z-gfxj/recording/chat-events.jsonl",
      "recordingSha256": "04d404a5eb138c882f566e2f8c82a43f494e59e3e20aa84c5d9e56615a7d354f",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-10-28T04-22-52-767Z-gfxj/metrics.json",
      "metricsSha256": "ee8be4d197bd469a8483c398ba683c43db873f931444901a490eaa77151c5e00",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1532,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T04:23:05.528Z",
            "sessionId": "90d33611-d069-4e0a-935d-eca913940f4a",
            "source": "first-token-detail",
            "tokensPerSec": 1465,
            "ttftMs": 11133
          },
          {
            "line": 125,
            "rx": "2026-10-28T04:24:53.688Z",
            "sessionId": "90d33611-d069-4e0a-935d-eca913940f4a",
            "source": "first-token-detail",
            "tokensPerSec": 1920,
            "ttftMs": 4035
          },
          {
            "line": 518,
            "rx": "2026-10-28T04:25:59.307Z",
            "sessionId": "90d33611-d069-4e0a-935d-eca913940f4a",
            "source": "first-token-detail",
            "tokensPerSec": 1599,
            "ttftMs": 4389
          },
          {
            "line": 537,
            "rx": "2026-10-28T04:26:05.563Z",
            "sessionId": "90d33611-d069-4e0a-935d-eca913940f4a",
            "source": "first-token-detail",
            "tokensPerSec": 1118,
            "ttftMs": 587
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-meeting-followup",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "meeting-followup",
        "repetition": 1
      },
      "trialId": "meeting-followup-mlx-qwen3-8-27b-q4-2026-10-28T04-29-49-275Z-4vcj",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-10-28T04-29-49-275Z-4vcj/recording/chat-events.jsonl",
      "recordingSha256": "f269fe6f408ac8144880d0bdcdaa88eafe59fecaadd7f8ae3488982950835719",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-10-28T04-29-49-275Z-4vcj/metrics.json",
      "metricsSha256": "ffef8a335b6af8550c5e5174ca72596be6f6275644c77ef12c90665e6ec29fb7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 862.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T04:30:06.291Z",
            "sessionId": "77ff256b-5283-4a81-a40c-32b80f83e1f4",
            "source": "first-token-detail",
            "tokensPerSec": 871,
            "ttftMs": 15374
          },
          {
            "line": 151,
            "rx": "2026-10-28T04:31:58.942Z",
            "sessionId": "77ff256b-5283-4a81-a40c-32b80f83e1f4",
            "source": "first-token-detail",
            "tokensPerSec": 854,
            "ttftMs": 2742
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-craftbook-spreadsheet-model",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-spreadsheet-model",
        "repetition": 1
      },
      "trialId": "craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-28T04-38-32-385Z-nyt0",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-28T04-38-32-385Z-nyt0/recording/chat-events.jsonl",
      "recordingSha256": "ed7535fe4afac5f3befa6ff88190b47c5a49d76e2f868d6e66170fa74b4a3a49",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-28T04-38-32-385Z-nyt0/metrics.json",
      "metricsSha256": "d57528b569bb8341d67fb63e5e9782310c90b6c5533b15e938c0b46df2b033b6",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 611,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T04:38:58.273Z",
            "sessionId": "97b31042-3940-4c4b-a224-58cf119a7f56",
            "source": "first-token-detail",
            "tokensPerSec": 630,
            "ttftMs": 19273
          },
          {
            "line": 341,
            "rx": "2026-10-28T04:40:03.280Z",
            "sessionId": "c97452c5-2ec7-484d-85e9-eda5512908e5",
            "source": "first-token-detail",
            "tokensPerSec": 611,
            "ttftMs": 14210
          },
          {
            "line": 4466,
            "rx": "2026-10-28T04:52:30.409Z",
            "sessionId": "f855d098-4f89-430e-b276-cb4af222cd65",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 16010
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-craftbook-spreadsheet-model",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-spreadsheet-model",
        "repetition": 1
      },
      "trialId": "craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-28T05-10-15-645Z-m954",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-28T05-10-15-645Z-m954/recording/chat-events.jsonl",
      "recordingSha256": "d0610244f2f7c57779ccb8a373bf85d5a7ffcbc61d83818ac05344c91b2e5407",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-28T05-10-15-645Z-m954/metrics.json",
      "metricsSha256": "8dd6743da2fa41957b9d757270bd1174ec31e310db6b017fa18042daaec42445",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1063,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T05:10:36.864Z",
            "sessionId": "29fdd75e-4ee1-4d65-992d-54cb7e8dcc2d",
            "source": "first-token-detail",
            "tokensPerSec": 1063,
            "ttftMs": 14674
          },
          {
            "line": 154,
            "rx": "2026-10-28T05:11:14.137Z",
            "sessionId": "2a97ee88-ed0e-462c-acee-e90b28cb1b6d",
            "source": "first-token-detail",
            "tokensPerSec": 1359,
            "ttftMs": 6882
          },
          {
            "line": 438,
            "rx": "2026-10-28T05:12:35.289Z",
            "sessionId": "add0afe1-20b7-4a1b-b3a5-ab82e65e6147",
            "source": "first-token-detail",
            "tokensPerSec": 1088,
            "ttftMs": 8036
          },
          {
            "line": 580,
            "rx": "2026-10-28T05:13:20.258Z",
            "sessionId": "1782fddc-40d9-4307-97da-ac3179bb602a",
            "source": "first-token-detail",
            "tokensPerSec": 1205,
            "ttftMs": 8061
          },
          {
            "line": 811,
            "rx": "2026-10-28T05:14:34.897Z",
            "sessionId": "39aa5cf3-59a3-44b2-a3ef-004628798a43",
            "source": "first-token-detail",
            "tokensPerSec": 988,
            "ttftMs": 7645
          },
          {
            "line": 1186,
            "rx": "2026-10-28T05:16:31.337Z",
            "sessionId": "e558a547-a3f0-4fd9-a5fb-34630ce2353a",
            "source": "first-token-detail",
            "tokensPerSec": 934,
            "ttftMs": 9088
          },
          {
            "line": 1422,
            "rx": "2026-10-28T05:17:44.818Z",
            "sessionId": "37e73094-764a-4dce-9f51-bf7c406483ac",
            "source": "first-token-detail",
            "tokensPerSec": 1044,
            "ttftMs": 7579
          },
          {
            "line": 3404,
            "rx": "2026-10-28T05:23:25.054Z",
            "sessionId": "d9307a6e-7c6d-4e3d-b65c-27372e002418",
            "source": "first-token-detail",
            "tokensPerSec": 1086,
            "ttftMs": 7681
          },
          {
            "line": 3514,
            "rx": "2026-10-28T05:23:47.300Z",
            "sessionId": "d97c15a6-cea7-499f-8854-17b855ef8cd6",
            "source": "first-token-detail",
            "tokensPerSec": 1062,
            "ttftMs": 4775
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-conflict-synthesis",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "conflict-synthesis",
        "repetition": 1
      },
      "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T05-24-23-449Z-rp3g",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T05-24-23-449Z-rp3g/recording/chat-events.jsonl",
      "recordingSha256": "caa83427b1b3882c54488dfefe54ca64a0be2a817564ab3e3c3bf25867e550e9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-28T05-24-23-449Z-rp3g/metrics.json",
      "metricsSha256": "6c14019ae9abd597a6376d15d27760e63210efc62c55d8f7330581d381a26afc",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 519,
        "sampleCount": 11,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 11,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T05:24:52.334Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 27237
          },
          {
            "line": 1134,
            "rx": "2026-10-28T05:28:09.541Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 568,
            "ttftMs": 8528
          },
          {
            "line": 1738,
            "rx": "2026-10-28T05:30:19.828Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 537,
            "ttftMs": 8849
          },
          {
            "line": 2050,
            "rx": "2026-10-28T05:31:43.093Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 522,
            "ttftMs": 11775
          },
          {
            "line": 2465,
            "rx": "2026-10-28T05:33:31.908Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 15165
          },
          {
            "line": 2910,
            "rx": "2026-10-28T05:35:34.596Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 17359
          },
          {
            "line": 3336,
            "rx": "2026-10-28T05:37:43.120Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 20365
          },
          {
            "line": 3775,
            "rx": "2026-10-28T05:40:02.969Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 495,
            "ttftMs": 24681
          },
          {
            "line": 3860,
            "rx": "2026-10-28T05:41:05.234Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 497,
            "ttftMs": 26696
          },
          {
            "line": 4272,
            "rx": "2026-10-28T05:43:39.471Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 495,
            "ttftMs": 30284
          },
          {
            "line": 4697,
            "rx": "2026-10-28T05:46:22.742Z",
            "sessionId": "ad6187d6-7ecf-4cfb-8eb6-3afd72a484ec",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 32911
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-conflict-synthesis",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "conflict-synthesis",
        "repetition": 1
      },
      "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T05-54-16-421Z-j9gb",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T05-54-16-421Z-j9gb/recording/chat-events.jsonl",
      "recordingSha256": "1bff4d6f6ef2db0472f25e7eb96234d806384bcce4ce08f1e886671c2eef2650",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-28T05-54-16-421Z-j9gb/metrics.json",
      "metricsSha256": "97caf6c8f9b315b0325c3b64a9680a91065bfe09008b9f6e6b75820d810b5df7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1082.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-28T05:54:34.015Z",
            "sessionId": "c7c0ed85-f6ac-4951-a6e7-0a54e6d12d3a",
            "source": "first-token-detail",
            "tokensPerSec": 1039,
            "ttftMs": 15960
          },
          {
            "line": 213,
            "rx": "2026-10-28T05:55:28.309Z",
            "sessionId": "c7c0ed85-f6ac-4951-a6e7-0a54e6d12d3a",
            "source": "first-token-detail",
            "tokensPerSec": 1368,
            "ttftMs": 4934
          },
          {
            "line": 350,
            "rx": "2026-10-28T05:56:21.108Z",
            "sessionId": "c7c0ed85-f6ac-4951-a6e7-0a54e6d12d3a",
            "source": "first-token-detail",
            "tokensPerSec": 1029,
            "ttftMs": 8007
          },
          {
            "line": 418,
            "rx": "2026-10-28T05:56:42.878Z",
            "sessionId": "c7c0ed85-f6ac-4951-a6e7-0a54e6d12d3a",
            "source": "first-token-detail",
            "tokensPerSec": 1126,
            "ttftMs": 9683
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-docblocks-theme-roundtrip",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "docblocks-theme-roundtrip",
        "repetition": 1
      },
      "trialId": "docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-28T06-05-32-061Z-39pc",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-28T06-05-32-061Z-39pc/recording/chat-events.jsonl",
      "recordingSha256": "816c822916b3c012a652011b945e978b3f0a3483d65e9b14491c6412155906cb",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-28T06-05-32-061Z-39pc/metrics.json",
      "metricsSha256": "80ba2096b33ad20ab4e30d37b93f22bc299b77a573f837b0ed0e9b5025fcc733",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1085,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-28T06:05:45.667Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 988,
            "ttftMs": 11978
          },
          {
            "line": 493,
            "rx": "2026-10-28T06:06:59.340Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 841,
            "ttftMs": 744
          },
          {
            "line": 1406,
            "rx": "2026-10-28T06:09:34.511Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 1186,
            "ttftMs": 5212
          },
          {
            "line": 1436,
            "rx": "2026-10-28T06:09:39.939Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 814,
            "ttftMs": 584
          },
          {
            "line": 1506,
            "rx": "2026-10-28T06:10:11.014Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 1085,
            "ttftMs": 6541
          },
          {
            "line": 1552,
            "rx": "2026-10-28T06:10:36.555Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 1172,
            "ttftMs": 6952
          },
          {
            "line": 1573,
            "rx": "2026-10-28T06:10:45.180Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 1026,
            "ttftMs": 495
          },
          {
            "line": 1610,
            "rx": "2026-10-28T06:10:55.313Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 1100,
            "ttftMs": 563
          },
          {
            "line": 1688,
            "rx": "2026-10-28T06:11:39.022Z",
            "sessionId": "be58517f-bad7-4b95-b041-c8e13bf8339e",
            "source": "first-token-detail",
            "tokensPerSec": 1136,
            "ttftMs": 9121
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-docblocks-theme-roundtrip",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "docblocks-theme-roundtrip",
        "repetition": 1
      },
      "trialId": "docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-28T06-18-05-860Z-y8gy",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-28T06-18-05-860Z-y8gy/recording/chat-events.jsonl",
      "recordingSha256": "f900988e44ea7eacb8a865e09847c01efb7285588f638fbbf3fe679b42668558",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-28T06-18-05-860Z-y8gy/metrics.json",
      "metricsSha256": "50235e53f0ecf5fd8186b7e8f13efdc38cceb48dcb72087bd1274ff8a5de0f7a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 511,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-28T06:18:25.221Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 473,
            "ttftMs": 17709
          },
          {
            "line": 178,
            "rx": "2026-10-28T06:18:52.289Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 369,
            "ttftMs": 1816
          },
          {
            "line": 385,
            "rx": "2026-10-28T06:19:36.199Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 577,
            "ttftMs": 8878
          },
          {
            "line": 450,
            "rx": "2026-10-28T06:20:04.261Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 612,
            "ttftMs": 6809
          },
          {
            "line": 543,
            "rx": "2026-10-28T06:20:37.385Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 550,
            "ttftMs": 9777
          },
          {
            "line": 609,
            "rx": "2026-10-28T06:21:10.490Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 528,
            "ttftMs": 12741
          },
          {
            "line": 668,
            "rx": "2026-10-28T06:21:48.829Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 509,
            "ttftMs": 15943
          },
          {
            "line": 743,
            "rx": "2026-10-28T06:22:37.026Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 510,
            "ttftMs": 18943
          },
          {
            "line": 806,
            "rx": "2026-10-28T06:23:30.378Z",
            "sessionId": "77453ec4-23d3-42f2-abe0-a973f2d4198b",
            "source": "first-token-detail",
            "tokensPerSec": 511,
            "ttftMs": 22083
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-craftbook-powerpoint-deck",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-powerpoint-deck",
        "repetition": 1
      },
      "trialId": "craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-28T06-24-33-379Z-5bn5",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-28T06-24-33-379Z-5bn5/recording/chat-events.jsonl",
      "recordingSha256": "6f35781a6470dc6c65d50829f9a5df36e91de97b0503316dbd0a8c190b34b613",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-28T06-24-33-379Z-5bn5/metrics.json",
      "metricsSha256": "22b274a7e98c93d5ee9c2d83f8e92354d40621d8c29fd392f924146db9b02027",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 929,
        "sampleCount": 50,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 50,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T06:24:55.372Z",
            "sessionId": "dc032ac9-01e6-4e37-a42d-ed57373c2046",
            "source": "first-token-detail",
            "tokensPerSec": 1105,
            "ttftMs": 15430
          },
          {
            "line": 156,
            "rx": "2026-10-28T06:25:31.578Z",
            "sessionId": "1e06c5a4-7ccd-443c-959c-a3ea157e2840",
            "source": "first-token-detail",
            "tokensPerSec": 1210,
            "ttftMs": 6617
          },
          {
            "line": 268,
            "rx": "2026-10-28T06:26:02.095Z",
            "sessionId": "5e22ff31-3373-4f3a-9b14-8f30ee22b27f",
            "source": "first-token-detail",
            "tokensPerSec": 1289,
            "ttftMs": 7150
          },
          {
            "line": 425,
            "rx": "2026-10-28T06:26:40.730Z",
            "sessionId": "472446ca-441a-49e7-a979-36376c454248",
            "source": "first-token-detail",
            "tokensPerSec": 1444,
            "ttftMs": 5785
          },
          {
            "line": 625,
            "rx": "2026-10-28T06:27:42.693Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1155,
            "ttftMs": 12097
          },
          {
            "line": 2417,
            "rx": "2026-10-28T06:34:06.950Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 337,
            "ttftMs": 513
          },
          {
            "line": 2628,
            "rx": "2026-10-28T06:34:40.639Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 461,
            "ttftMs": 611
          },
          {
            "line": 3278,
            "rx": "2026-10-28T06:36:29.089Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 493,
            "ttftMs": 604
          },
          {
            "line": 3649,
            "rx": "2026-10-28T06:37:29.318Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 824,
            "ttftMs": 518
          },
          {
            "line": 5193,
            "rx": "2026-10-28T06:42:09.204Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1069,
            "ttftMs": 18946
          },
          {
            "line": 5295,
            "rx": "2026-10-28T06:42:31.058Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 828,
            "ttftMs": 548
          },
          {
            "line": 5422,
            "rx": "2026-10-28T06:43:23.003Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 950,
            "ttftMs": 27718
          },
          {
            "line": 5757,
            "rx": "2026-10-28T06:44:42.811Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1036,
            "ttftMs": 21706
          },
          {
            "line": 5922,
            "rx": "2026-10-28T06:46:08.919Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 918,
            "ttftMs": 31671
          },
          {
            "line": 6221,
            "rx": "2026-10-28T06:47:22.004Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 985,
            "ttftMs": 25006
          },
          {
            "line": 6405,
            "rx": "2026-10-28T06:51:24.748Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 1837,
            "ttftMs": 7312
          },
          {
            "line": 7679,
            "rx": "2026-10-28T06:54:45.897Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 365,
            "ttftMs": 446
          },
          {
            "line": 7757,
            "rx": "2026-10-28T06:55:05.290Z",
            "sessionId": "d5b7b27c-0180-41fb-8118-dfa37c8dbb78",
            "source": "first-token-detail",
            "tokensPerSec": 1014,
            "ttftMs": 18595
          },
          {
            "line": 7943,
            "rx": "2026-10-28T06:58:21.866Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 607,
            "ttftMs": 412
          },
          {
            "line": 8069,
            "rx": "2026-10-28T06:58:40.094Z",
            "sessionId": "d5b7b27c-0180-41fb-8118-dfa37c8dbb78",
            "source": "first-token-detail",
            "tokensPerSec": 992,
            "ttftMs": 306
          },
          {
            "line": 8098,
            "rx": "2026-10-28T06:58:42.827Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 1020,
            "ttftMs": 550
          },
          {
            "line": 8281,
            "rx": "2026-10-28T06:59:05.422Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 337
          },
          {
            "line": 8461,
            "rx": "2026-10-28T06:59:30.223Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 714,
            "ttftMs": 566
          },
          {
            "line": 8578,
            "rx": "2026-10-28T06:59:44.421Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 424,
            "ttftMs": 392
          },
          {
            "line": 8729,
            "rx": "2026-10-28T07:00:03.968Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 386,
            "ttftMs": 429
          },
          {
            "line": 8959,
            "rx": "2026-10-28T07:00:42.540Z",
            "sessionId": "4e186da1-0a3a-454e-a46d-c2f0171c034c",
            "source": "first-token-detail",
            "tokensPerSec": 1240,
            "ttftMs": 16217
          },
          {
            "line": 9205,
            "rx": "2026-10-28T07:01:21.365Z",
            "sessionId": "4e186da1-0a3a-454e-a46d-c2f0171c034c",
            "source": "first-token-detail",
            "tokensPerSec": 661,
            "ttftMs": 756
          },
          {
            "line": 9344,
            "rx": "2026-10-28T07:01:41.609Z",
            "sessionId": "4e186da1-0a3a-454e-a46d-c2f0171c034c",
            "source": "first-token-detail",
            "tokensPerSec": 416,
            "ttftMs": 916
          },
          {
            "line": 9538,
            "rx": "2026-10-28T07:02:11.660Z",
            "sessionId": "4e186da1-0a3a-454e-a46d-c2f0171c034c",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 828
          },
          {
            "line": 9825,
            "rx": "2026-10-28T07:03:00.450Z",
            "sessionId": "d5b7b27c-0180-41fb-8118-dfa37c8dbb78",
            "source": "first-token-detail",
            "tokensPerSec": 407,
            "ttftMs": 778
          },
          {
            "line": 9928,
            "rx": "2026-10-28T07:03:29.456Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 779,
            "ttftMs": 16037
          },
          {
            "line": 10259,
            "rx": "2026-10-28T07:04:28.305Z",
            "sessionId": "5872b07a-6fcd-4085-8a3b-b7b8b32ba2f8",
            "source": "first-token-detail",
            "tokensPerSec": 826,
            "ttftMs": 42304
          },
          {
            "line": 10509,
            "rx": "2026-10-28T07:05:24.660Z",
            "sessionId": "0de4bc0c-bb32-4407-b2a7-a80587c545b3",
            "source": "first-token-detail",
            "tokensPerSec": 1123,
            "ttftMs": 8842
          },
          {
            "line": 10920,
            "rx": "2026-10-28T07:07:03.066Z",
            "sessionId": "14ed3159-4eac-45d1-946f-65833df049a5",
            "source": "first-token-detail",
            "tokensPerSec": 1250,
            "ttftMs": 7240
          },
          {
            "line": 11081,
            "rx": "2026-10-28T07:08:44.535Z",
            "sessionId": "cc3d8af9-2b04-4634-bd56-d9d43a768ad4",
            "source": "first-token-detail",
            "tokensPerSec": 1781,
            "ttftMs": 8218
          },
          {
            "line": 12490,
            "rx": "2026-10-28T07:12:24.583Z",
            "sessionId": "cc3d8af9-2b04-4634-bd56-d9d43a768ad4",
            "source": "first-token-detail",
            "tokensPerSec": 862,
            "ttftMs": 9712
          },
          {
            "line": 12756,
            "rx": "2026-10-28T07:16:29.046Z",
            "sessionId": "80ee5704-0321-40c6-9047-50814980ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1438,
            "ttftMs": 22666
          },
          {
            "line": 13003,
            "rx": "2026-10-28T07:17:34.998Z",
            "sessionId": "d5b7b27c-0180-41fb-8118-dfa37c8dbb78",
            "source": "first-token-detail",
            "tokensPerSec": 399,
            "ttftMs": 395
          },
          {
            "line": 13102,
            "rx": "2026-10-28T07:20:50.337Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 1752,
            "ttftMs": 8918
          },
          {
            "line": 13308,
            "rx": "2026-10-28T07:21:19.385Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 904,
            "ttftMs": 498
          },
          {
            "line": 13407,
            "rx": "2026-10-28T07:21:33.276Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 511,
            "ttftMs": 296
          },
          {
            "line": 13550,
            "rx": "2026-10-28T07:21:54.909Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 696,
            "ttftMs": 598
          },
          {
            "line": 13682,
            "rx": "2026-10-28T07:22:14.770Z",
            "sessionId": "f69b9f9f-0fa2-4ca6-a146-27abc7b75943",
            "source": "first-token-detail",
            "tokensPerSec": 410,
            "ttftMs": 358
          },
          {
            "line": 13775,
            "rx": "2026-10-28T07:22:38.366Z",
            "sessionId": "0022280c-0ace-494c-8626-f60d158f29f2",
            "source": "first-token-detail",
            "tokensPerSec": 1526,
            "ttftMs": 7124
          },
          {
            "line": 13972,
            "rx": "2026-10-28T07:23:21.069Z",
            "sessionId": "debd8963-986b-41b2-9d33-decad747f196",
            "source": "first-token-detail",
            "tokensPerSec": 1469,
            "ttftMs": 4848
          },
          {
            "line": 14096,
            "rx": "2026-10-28T07:23:54.015Z",
            "sessionId": "8ad13437-9802-454b-be97-c7672c46ad85",
            "source": "first-token-detail",
            "tokensPerSec": 1226,
            "ttftMs": 7780
          },
          {
            "line": 14423,
            "rx": "2026-10-28T07:25:09.190Z",
            "sessionId": "eb8643fe-b6f8-444a-9d47-f59d152f19c4",
            "source": "first-token-detail",
            "tokensPerSec": 1113,
            "ttftMs": 7971
          },
          {
            "line": 14593,
            "rx": "2026-10-28T07:26:01.805Z",
            "sessionId": "090c6bca-28c6-448f-aae5-a2134f6be62f",
            "source": "first-token-detail",
            "tokensPerSec": 961,
            "ttftMs": 15003
          },
          {
            "line": 15167,
            "rx": "2026-10-28T07:29:09.842Z",
            "sessionId": "090c6bca-28c6-448f-aae5-a2134f6be62f",
            "source": "first-token-detail",
            "tokensPerSec": 940,
            "ttftMs": 543
          },
          {
            "line": 17012,
            "rx": "2026-10-28T07:34:09.274Z",
            "sessionId": "090c6bca-28c6-448f-aae5-a2134f6be62f",
            "source": "first-token-detail",
            "tokensPerSec": 326,
            "ttftMs": 487
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-craftbook-powerpoint-deck",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-powerpoint-deck",
        "repetition": 1
      },
      "trialId": "craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-28T07-39-23-343Z-881f",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-28T07-39-23-343Z-881f/recording/chat-events.jsonl",
      "recordingSha256": "90fff2607ad1bcc6889ae7fcd74830fc957532d8492edf951f37084a64a5074c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-28T07-39-23-343Z-881f/metrics.json",
      "metricsSha256": "f2fa17528a8a21e8e50eb6d5301fba76680c9cf13a952eecf16cb8a45329b857",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 524.5,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-28T07:39:53.186Z",
            "sessionId": "56ddeb4c-a55a-4dae-ae3f-fd80a5f6906c",
            "source": "first-token-detail",
            "tokensPerSec": 620,
            "ttftMs": 23269
          },
          {
            "line": 2855,
            "rx": "2026-10-28T07:48:35.486Z",
            "sessionId": "8b680434-22a8-47fe-b5de-e9b3ed2b0ee7",
            "source": "first-token-detail",
            "tokensPerSec": 539,
            "ttftMs": 15416
          },
          {
            "line": 3396,
            "rx": "2026-10-28T07:50:27.847Z",
            "sessionId": "fc2b84d6-ec0d-4724-8cb2-cb7c5aab4dd6",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 17702
          },
          {
            "line": 3429,
            "rx": "2026-10-28T07:50:38.770Z",
            "sessionId": "fc2b84d6-ec0d-4724-8cb2-cb7c5aab4dd6",
            "source": "first-token-detail",
            "tokensPerSec": 211,
            "ttftMs": 929
          },
          {
            "line": 3673,
            "rx": "2026-10-28T07:51:36.806Z",
            "sessionId": "79c8f47d-41fe-4a90-9e7d-81cbf833563c",
            "source": "first-token-detail",
            "tokensPerSec": 533,
            "ttftMs": 16649
          },
          {
            "line": 5524,
            "rx": "2026-10-28T07:57:59.531Z",
            "sessionId": "fd50d7d7-9ed5-41d0-a9b4-8374bc64fe56",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 53642
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-wikipedia-research-brief",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "wikipedia-research-brief",
        "repetition": 1
      },
      "trialId": "wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-28T08-15-32-657Z-xw1n",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-28T08-15-32-657Z-xw1n/recording/chat-events.jsonl",
      "recordingSha256": "2338ca9deee2fbdffd9eca48b58cde3bc5c705414c03188c7925c8e5953dd1a0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-28T08-15-32-657Z-xw1n/metrics.json",
      "metricsSha256": "f7ab61d37e2d7ec325c813513d4a3aa17137e1a7645cacd338f4011790627ec0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 566.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-10-28T08:15:46.113Z",
            "sessionId": "3ec1e505-4d31-49fd-b6ed-d71667e3430e",
            "source": "first-token-detail",
            "tokensPerSec": 541,
            "ttftMs": 11754
          },
          {
            "line": 630,
            "rx": "2026-10-28T08:17:37.556Z",
            "sessionId": "3ec1e505-4d31-49fd-b6ed-d71667e3430e",
            "source": "first-token-detail",
            "tokensPerSec": 622,
            "ttftMs": 8270
          },
          {
            "line": 832,
            "rx": "2026-10-28T08:18:28.361Z",
            "sessionId": "3ec1e505-4d31-49fd-b6ed-d71667e3430e",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 8866
          },
          {
            "line": 916,
            "rx": "2026-10-28T08:19:08.038Z",
            "sessionId": "3ec1e505-4d31-49fd-b6ed-d71667e3430e",
            "source": "first-token-detail",
            "tokensPerSec": 563,
            "ttftMs": 13383
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-wikipedia-research-brief",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "wikipedia-research-brief",
        "repetition": 1
      },
      "trialId": "wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-28T08-31-49-358Z-pkqi",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-28T08-31-49-358Z-pkqi/recording/chat-events.jsonl",
      "recordingSha256": "7441c7d995b623514943b911747a56363062b5f64119a2dd3b2eac95777a0ba0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-28T08-31-49-358Z-pkqi/metrics.json",
      "metricsSha256": "84926bff0947fa876f36c3419c4ad607113fa0ed967cbc9e445c9925f3c9a28a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1010,
        "sampleCount": 16,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 16,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-10-28T08:32:01.281Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1127,
            "ttftMs": 10145
          },
          {
            "line": 180,
            "rx": "2026-10-28T08:32:52.436Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1149,
            "ttftMs": 6585
          },
          {
            "line": 572,
            "rx": "2026-10-28T08:35:16.397Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1231,
            "ttftMs": 9938
          },
          {
            "line": 705,
            "rx": "2026-10-28T08:36:19.583Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1126,
            "ttftMs": 12847
          },
          {
            "line": 805,
            "rx": "2026-10-28T08:37:16.644Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1108,
            "ttftMs": 14659
          },
          {
            "line": 929,
            "rx": "2026-10-28T08:37:58.869Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1323,
            "ttftMs": 1623
          },
          {
            "line": 1188,
            "rx": "2026-10-28T08:39:16.824Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1020,
            "ttftMs": 19301
          },
          {
            "line": 1274,
            "rx": "2026-10-28T08:39:44.357Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 1622
          },
          {
            "line": 1400,
            "rx": "2026-10-28T08:40:25.415Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 901,
            "ttftMs": 2483
          },
          {
            "line": 1673,
            "rx": "2026-10-28T08:50:42.665Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 1489,
            "ttftMs": 25689
          },
          {
            "line": 3070,
            "rx": "2026-10-28T08:54:30.066Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 773,
            "ttftMs": 10799
          },
          {
            "line": 3299,
            "rx": "2026-10-28T08:55:45.061Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 828,
            "ttftMs": 43862
          },
          {
            "line": 3566,
            "rx": "2026-10-28T08:57:46.009Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 843,
            "ttftMs": 46970
          },
          {
            "line": 3691,
            "rx": "2026-10-28T08:58:27.451Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 680,
            "ttftMs": 3036
          },
          {
            "line": 3707,
            "rx": "2026-10-28T08:58:31.341Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 1895
          },
          {
            "line": 3906,
            "rx": "2026-10-28T09:01:07.613Z",
            "sessionId": "d3a54415-ded8-40ef-9dbf-be9c65d66355",
            "source": "first-token-detail",
            "tokensPerSec": 782,
            "ttftMs": 2513
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-craftbook-week-plan",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-week-plan",
        "repetition": 1
      },
      "trialId": "craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-28T09-02-14-313Z-9xn3",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-28T09-02-14-313Z-9xn3/recording/chat-events.jsonl",
      "recordingSha256": "566c48c12d339b319f1401d27b2773c10167e5f73f6d3a7877d93e6c5e4d867b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-28T09-02-14-313Z-9xn3/metrics.json",
      "metricsSha256": "a6fe2b6821700de359ed82bbe01dd5200fa6d7af70e83e0c81f5078fc4684953",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 565,
        "sampleCount": 11,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 11,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 36,
            "rx": "2026-10-28T09:02:53.513Z",
            "sessionId": "f022b7c4-0618-4b2f-ac71-fc8d54456fc7",
            "source": "first-token-detail",
            "tokensPerSec": 565,
            "ttftMs": 32621
          },
          {
            "line": 4301,
            "rx": "2026-10-28T09:15:47.311Z",
            "sessionId": "f022b7c4-0618-4b2f-ac71-fc8d54456fc7",
            "source": "first-token-detail",
            "tokensPerSec": 499,
            "ttftMs": 23182
          },
          {
            "line": 9556,
            "rx": "2026-10-28T09:35:17.792Z",
            "sessionId": "f022b7c4-0618-4b2f-ac71-fc8d54456fc7",
            "source": "first-token-detail",
            "tokensPerSec": 211,
            "ttftMs": 1647
          },
          {
            "line": 14924,
            "rx": "2026-10-28T09:57:14.212Z",
            "sessionId": "f022b7c4-0618-4b2f-ac71-fc8d54456fc7",
            "source": "first-token-detail",
            "tokensPerSec": 450,
            "ttftMs": 67039
          },
          {
            "line": 16044,
            "rx": "2026-10-28T10:01:22.307Z",
            "sessionId": "6ac11a5e-e485-4257-ab33-fbafbc88a548",
            "source": "first-token-detail",
            "tokensPerSec": 578,
            "ttftMs": 15067
          },
          {
            "line": 16716,
            "rx": "2026-10-28T10:03:49.139Z",
            "sessionId": "6604aaf3-e4ec-4028-9c60-7b420e303ac8",
            "source": "first-token-detail",
            "tokensPerSec": 555,
            "ttftMs": 16893
          },
          {
            "line": 18387,
            "rx": "2026-10-28T10:09:54.363Z",
            "sessionId": "881e5a75-5289-4958-8f50-1b1ca94fe5cb",
            "source": "first-token-detail",
            "tokensPerSec": 847,
            "ttftMs": 7021
          },
          {
            "line": 18646,
            "rx": "2026-10-28T10:11:49.541Z",
            "sessionId": "cd812d17-9361-4597-a4bb-bac30268ac83",
            "source": "first-token-detail",
            "tokensPerSec": 874,
            "ttftMs": 7081
          },
          {
            "line": 20306,
            "rx": "2026-10-28T10:17:34.984Z",
            "sessionId": "b0c07ebe-ed88-41cb-b7f6-6ce48fbec88d",
            "source": "first-token-detail",
            "tokensPerSec": 552,
            "ttftMs": 12398
          },
          {
            "line": 20489,
            "rx": "2026-10-28T10:19:20.973Z",
            "sessionId": "f73c1554-a012-4529-8c43-348a02c97065",
            "source": "first-token-detail",
            "tokensPerSec": 853,
            "ttftMs": 8344
          },
          {
            "line": 24264,
            "rx": "2026-10-28T10:30:56.778Z",
            "sessionId": "0f6d557e-b8cb-43ff-9360-84152474bb61",
            "source": "first-token-detail",
            "tokensPerSec": 784,
            "ttftMs": 8870
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-craftbook-week-plan",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-week-plan",
        "repetition": 1
      },
      "trialId": "craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-28T11-03-04-929Z-q9pi",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-28T11-03-04-929Z-q9pi/recording/chat-events.jsonl",
      "recordingSha256": "bea19c0424f9f2ec45c90ebdc85ed5d381eae010c4e02a257f9f307dc405a6dc",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-28T11-03-04-929Z-q9pi/metrics.json",
      "metricsSha256": "0fa39c2d42d37a5d9d1b78e4ab903728e059f642c53bfa1159c0409bfbb6ab48",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1161,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T11:03:30.658Z",
            "sessionId": "aefccd55-75b3-45c8-b6b7-ba60fa06a21b",
            "source": "first-token-detail",
            "tokensPerSec": 970,
            "ttftMs": 19175
          },
          {
            "line": 172,
            "rx": "2026-10-28T11:04:12.414Z",
            "sessionId": "aefccd55-75b3-45c8-b6b7-ba60fa06a21b",
            "source": "first-token-detail",
            "tokensPerSec": 1158,
            "ttftMs": 10925
          },
          {
            "line": 393,
            "rx": "2026-10-28T11:05:02.644Z",
            "sessionId": "90e85c68-b5d5-40a1-a703-409f1df43dcf",
            "source": "first-token-detail",
            "tokensPerSec": 1343,
            "ttftMs": 6069
          },
          {
            "line": 494,
            "rx": "2026-10-28T11:05:34.077Z",
            "sessionId": "1ae7676b-6efc-412c-9552-5335665b1bb4",
            "source": "first-token-detail",
            "tokensPerSec": 1164,
            "ttftMs": 7601
          },
          {
            "line": 628,
            "rx": "2026-10-28T11:06:07.294Z",
            "sessionId": "80b74878-817b-4b97-8498-272b7b44bff5",
            "source": "first-token-detail",
            "tokensPerSec": 1284,
            "ttftMs": 5745
          },
          {
            "line": 881,
            "rx": "2026-10-28T11:07:03.116Z",
            "sessionId": "5f67b439-9e59-42dc-a816-9a04dd9e9765",
            "source": "first-token-detail",
            "tokensPerSec": 1161,
            "ttftMs": 6550
          },
          {
            "line": 2786,
            "rx": "2026-10-28T11:12:28.208Z",
            "sessionId": "4c9fbbea-0ecc-40cc-bd95-97cf575c0085",
            "source": "first-token-detail",
            "tokensPerSec": 1236,
            "ttftMs": 6456
          },
          {
            "line": 2930,
            "rx": "2026-10-28T11:13:07.031Z",
            "sessionId": "75fae9e8-2818-4138-8911-318588b985ed",
            "source": "first-token-detail",
            "tokensPerSec": 1138,
            "ttftMs": 12846
          },
          {
            "line": 3857,
            "rx": "2026-10-28T11:15:53.919Z",
            "sessionId": "75fae9e8-2818-4138-8911-318588b985ed",
            "source": "first-token-detail",
            "tokensPerSec": 1107,
            "ttftMs": 9394
          },
          {
            "line": 3925,
            "rx": "2026-10-28T11:16:14.005Z",
            "sessionId": "ed097115-bb27-4921-9f03-66b235ef3e30",
            "source": "first-token-detail",
            "tokensPerSec": 164,
            "ttftMs": 29796
          },
          {
            "line": 3950,
            "rx": "2026-10-28T11:16:28.185Z",
            "sessionId": "ed097115-bb27-4921-9f03-66b235ef3e30",
            "source": "first-token-detail",
            "tokensPerSec": 1410,
            "ttftMs": 3463
          },
          {
            "line": 3981,
            "rx": "2026-10-28T11:16:47.447Z",
            "sessionId": "ed097115-bb27-4921-9f03-66b235ef3e30",
            "source": "first-token-detail",
            "tokensPerSec": 1268,
            "ttftMs": 7043
          },
          {
            "line": 4031,
            "rx": "2026-10-28T11:17:13.180Z",
            "sessionId": "ed097115-bb27-4921-9f03-66b235ef3e30",
            "source": "first-token-detail",
            "tokensPerSec": 1095,
            "ttftMs": 8260
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-craftbook-ab-test-readout",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-ab-test-readout",
        "repetition": 1
      },
      "trialId": "craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-28T11-52-16-787Z-zk09",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-28T11-52-16-787Z-zk09/recording/chat-events.jsonl",
      "recordingSha256": "24369438bb6a518f874d60d6d15d042523ec25a14c7c37585607ffb1d9ebb081",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-28T11-52-16-787Z-zk09/metrics.json",
      "metricsSha256": "ba0d2eae9668d53fe740cd591da10c66c85fa6cf966857254bbfa1574b717f22",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1141,
        "sampleCount": 19,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 19,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-28T11:52:36.372Z",
            "sessionId": "7534629f-64db-4bab-ba56-929483fb88b8",
            "source": "first-token-detail",
            "tokensPerSec": 1747,
            "ttftMs": 12831
          },
          {
            "line": 172,
            "rx": "2026-10-28T11:53:08.281Z",
            "sessionId": "7534629f-64db-4bab-ba56-929483fb88b8",
            "source": "first-token-detail",
            "tokensPerSec": 1493,
            "ttftMs": 9686
          },
          {
            "line": 662,
            "rx": "2026-10-28T11:54:34.985Z",
            "sessionId": "04a6284e-cd81-4acf-82d5-1463dcdec1cd",
            "source": "first-token-detail",
            "tokensPerSec": 1355,
            "ttftMs": 6301
          },
          {
            "line": 928,
            "rx": "2026-10-28T11:55:31.347Z",
            "sessionId": "aeeebe5a-35bc-4f18-a12a-08271062445d",
            "source": "first-token-detail",
            "tokensPerSec": 1203,
            "ttftMs": 7709
          },
          {
            "line": 1060,
            "rx": "2026-10-28T11:56:05.408Z",
            "sessionId": "586ef419-25d6-43b1-ac9f-3d19c2e314be",
            "source": "first-token-detail",
            "tokensPerSec": 1141,
            "ttftMs": 6796
          },
          {
            "line": 1204,
            "rx": "2026-10-28T11:56:41.763Z",
            "sessionId": "04a3c430-2329-4df5-a4c8-ac081da615d8",
            "source": "first-token-detail",
            "tokensPerSec": 993,
            "ttftMs": 8143
          },
          {
            "line": 1338,
            "rx": "2026-10-28T11:57:16.477Z",
            "sessionId": "6ddfac90-648d-4b79-8905-c93b9c9345c6",
            "source": "first-token-detail",
            "tokensPerSec": 1018,
            "ttftMs": 7829
          },
          {
            "line": 1537,
            "rx": "2026-10-28T11:57:53.599Z",
            "sessionId": "6ddfac90-648d-4b79-8905-c93b9c9345c6",
            "source": "first-token-detail",
            "tokensPerSec": 536,
            "ttftMs": 1003
          },
          {
            "line": 1871,
            "rx": "2026-10-28T11:59:12.044Z",
            "sessionId": "07f1aa7f-3451-4018-9a5f-47dadc1a78eb",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 8333
          },
          {
            "line": 1991,
            "rx": "2026-10-28T11:59:45.267Z",
            "sessionId": "853b692b-c768-4ce9-98bd-569d147bf1ee",
            "source": "first-token-detail",
            "tokensPerSec": 1245,
            "ttftMs": 6607
          },
          {
            "line": 2084,
            "rx": "2026-10-28T12:00:06.141Z",
            "sessionId": "853b692b-c768-4ce9-98bd-569d147bf1ee",
            "source": "first-token-detail",
            "tokensPerSec": 642,
            "ttftMs": 957
          },
          {
            "line": 2210,
            "rx": "2026-10-28T12:00:41.474Z",
            "sessionId": "b59bbf71-5d5d-4528-996b-3caaebb2ac17",
            "source": "first-token-detail",
            "tokensPerSec": 1120,
            "ttftMs": 7727
          },
          {
            "line": 2452,
            "rx": "2026-10-28T12:01:50.469Z",
            "sessionId": "3a3e7f19-657d-4d6f-9c64-876557c3cee7",
            "source": "first-token-detail",
            "tokensPerSec": 1270,
            "ttftMs": 6694
          },
          {
            "line": 2603,
            "rx": "2026-10-28T12:02:23.120Z",
            "sessionId": "3a3e7f19-657d-4d6f-9c64-876557c3cee7",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 983
          },
          {
            "line": 4618,
            "rx": "2026-10-28T12:08:16.654Z",
            "sessionId": "2606c5d2-c3c1-4f22-9aba-5b3f10e28207",
            "source": "first-token-detail",
            "tokensPerSec": 1186,
            "ttftMs": 8372
          },
          {
            "line": 4908,
            "rx": "2026-10-28T12:09:00.899Z",
            "sessionId": "2606c5d2-c3c1-4f22-9aba-5b3f10e28207",
            "source": "first-token-detail",
            "tokensPerSec": 386,
            "ttftMs": 349
          },
          {
            "line": 5502,
            "rx": "2026-10-28T12:10:35.799Z",
            "sessionId": "97821c43-6afa-451e-9d73-88bef7ee2ebb",
            "source": "first-token-detail",
            "tokensPerSec": 824,
            "ttftMs": 86798
          },
          {
            "line": 5714,
            "rx": "2026-10-28T12:12:29.026Z",
            "sessionId": "ef187765-c478-4db0-8063-52e903666c69",
            "source": "first-token-detail",
            "tokensPerSec": 1827,
            "ttftMs": 5044
          },
          {
            "line": 5856,
            "rx": "2026-10-28T12:13:00.838Z",
            "sessionId": "c0f5a648-32e7-4dba-8253-4f67a538100a",
            "source": "first-token-detail",
            "tokensPerSec": 1768,
            "ttftMs": 6298
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-craftbook-ab-test-readout",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-ab-test-readout",
        "repetition": 1
      },
      "trialId": "craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-28T13-37-19-431Z-r3n3",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-28T13-37-19-431Z-r3n3/recording/chat-events.jsonl",
      "recordingSha256": "facc510d779e18f7aa477227a760cad50d5a1d9ff025168b6005e201dad94dbb",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-28T13-37-19-431Z-r3n3/metrics.json",
      "metricsSha256": "0e4ebd9fa3b3dca7261f6788bfb001fbac00f5a501413be1a80ea6a9ad0c9b12",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 504,
        "sampleCount": 8,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 8,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-28T13:37:47.968Z",
            "sessionId": "4ab16bcf-57d1-47b9-8474-f8d62ff1f30c",
            "source": "first-token-detail",
            "tokensPerSec": 802,
            "ttftMs": 22067
          },
          {
            "line": 363,
            "rx": "2026-10-28T13:39:05.462Z",
            "sessionId": "4ab16bcf-57d1-47b9-8474-f8d62ff1f30c",
            "source": "first-token-detail",
            "tokensPerSec": 645,
            "ttftMs": 24424
          },
          {
            "line": 1795,
            "rx": "2026-10-28T13:43:24.691Z",
            "sessionId": "33ff89b9-01a5-4719-834b-01ddddce1d99",
            "source": "first-token-detail",
            "tokensPerSec": 452,
            "ttftMs": 18675
          },
          {
            "line": 2612,
            "rx": "2026-10-28T13:46:14.455Z",
            "sessionId": "92a2f3d2-6746-4712-ac11-8c059ccc0c50",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 18381
          },
          {
            "line": 3533,
            "rx": "2026-10-28T13:49:15.899Z",
            "sessionId": "c6500b0c-f76a-4408-aa73-a212f267f1c1",
            "source": "first-token-detail",
            "tokensPerSec": 532,
            "ttftMs": 9725
          },
          {
            "line": 4277,
            "rx": "2026-10-28T13:51:41.890Z",
            "sessionId": "69422676-9fae-4055-bcfa-84a998534d92",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 10660
          },
          {
            "line": 5696,
            "rx": "2026-10-28T13:56:18.898Z",
            "sessionId": "6d98fcea-430b-4cca-b5bb-60cae21a8587",
            "source": "first-token-detail",
            "tokensPerSec": 498,
            "ttftMs": 17631
          },
          {
            "line": 5853,
            "rx": "2026-10-28T13:57:11.237Z",
            "sessionId": "894d8331-8495-4a24-9ffd-be56892a3edd",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 25565
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-craftbook-annotated-bibliography",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-annotated-bibliography",
        "repetition": 1
      },
      "trialId": "craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-28T13-59-53-091Z-9lcu",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-28T13-59-53-091Z-9lcu/recording/chat-events.jsonl",
      "recordingSha256": "23a2b48316986a2e6d9ffe0bd3f74ef9acb600868cbe238ace338aee69e0e90a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-28T13-59-53-091Z-9lcu/metrics.json",
      "metricsSha256": "a2748c91a3c1e1053bd601b49b38cb2d209b647b86e7b0f75ea7e6c805636a82",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1099,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T14:00:14.961Z",
            "sessionId": "00520341-3704-4035-8ec6-c34c1cd41793",
            "source": "first-token-detail",
            "tokensPerSec": 916,
            "ttftMs": 15316
          },
          {
            "line": 247,
            "rx": "2026-10-28T14:01:05.588Z",
            "sessionId": "b7d3915a-9d47-4e17-8eba-36fdd8be6945",
            "source": "first-token-detail",
            "tokensPerSec": 989,
            "ttftMs": 10898
          },
          {
            "line": 468,
            "rx": "2026-10-28T14:02:11.840Z",
            "sessionId": "7dcb1f5d-8d0d-42fa-8742-ce07e6698561",
            "source": "first-token-detail",
            "tokensPerSec": 1239,
            "ttftMs": 7131
          },
          {
            "line": 695,
            "rx": "2026-10-28T14:03:16.810Z",
            "sessionId": "d9610773-0778-4e46-af1a-dcf29238c1fa",
            "source": "first-token-detail",
            "tokensPerSec": 1209,
            "ttftMs": 7092
          },
          {
            "line": 861,
            "rx": "2026-10-28T14:04:02.921Z",
            "sessionId": "7a9c9749-a290-4422-8025-bdd4c6881e37",
            "source": "first-token-detail",
            "tokensPerSec": 1095,
            "ttftMs": 8225
          },
          {
            "line": 1012,
            "rx": "2026-10-28T14:04:45.833Z",
            "sessionId": "735fb111-a67b-4369-bb05-4aee47e9f069",
            "source": "first-token-detail",
            "tokensPerSec": 1197,
            "ttftMs": 6172
          },
          {
            "line": 1134,
            "rx": "2026-10-28T14:05:17.991Z",
            "sessionId": "5e6e8833-3f34-4845-942d-4ff1f0b33cfc",
            "source": "first-token-detail",
            "tokensPerSec": 989,
            "ttftMs": 8369
          },
          {
            "line": 1347,
            "rx": "2026-10-28T14:06:26.235Z",
            "sessionId": "40adfbb1-dbd8-4f17-8945-6d6a670a7f1a",
            "source": "first-token-detail",
            "tokensPerSec": 1165,
            "ttftMs": 6458
          },
          {
            "line": 1828,
            "rx": "2026-10-28T14:07:58.093Z",
            "sessionId": "0e70ffee-4025-4a54-965b-aad0de430ea6",
            "source": "first-token-detail",
            "tokensPerSec": 1023,
            "ttftMs": 8223
          },
          {
            "line": 2021,
            "rx": "2026-10-28T14:08:46.670Z",
            "sessionId": "09daebfa-23ce-41df-950f-d903f3b4c233",
            "source": "first-token-detail",
            "tokensPerSec": 1112,
            "ttftMs": 6879
          },
          {
            "line": 2147,
            "rx": "2026-10-28T14:09:22.372Z",
            "sessionId": "09daebfa-23ce-41df-950f-d903f3b4c233",
            "source": "first-token-detail",
            "tokensPerSec": 1099,
            "ttftMs": 9703
          },
          {
            "line": 2960,
            "rx": "2026-10-28T14:11:43.610Z",
            "sessionId": "0ef302c7-5ae9-4381-a207-eb210ecf4da5",
            "source": "first-token-detail",
            "tokensPerSec": 1032,
            "ttftMs": 8771
          },
          {
            "line": 3104,
            "rx": "2026-10-28T14:12:26.677Z",
            "sessionId": "1d376b3d-dc34-48c7-8844-a7d36dcdc03d",
            "source": "first-token-detail",
            "tokensPerSec": 1199,
            "ttftMs": 6886
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-craftbook-annotated-bibliography",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-annotated-bibliography",
        "repetition": 1
      },
      "trialId": "craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-28T14-16-16-509Z-tygs",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-28T14-16-16-509Z-tygs/recording/chat-events.jsonl",
      "recordingSha256": "88ff37b24077fc70841f6ceb62f4638e18d0ae882e9ebd830143f54ae090eec9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-28T14-16-16-509Z-tygs/metrics.json",
      "metricsSha256": "caeef7a2a349392ab4d7ba7ad93dde174a7d9727a21cd47ddb9f7b76ddd0d2e9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 647,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T14:16:40.283Z",
            "sessionId": "8b927bb9-bf62-4bf6-afef-899533ceacd2",
            "source": "first-token-detail",
            "tokensPerSec": 715,
            "ttftMs": 17251
          },
          {
            "line": 209,
            "rx": "2026-10-28T14:17:25.687Z",
            "sessionId": "9feb1e14-7d79-4258-801f-b8a8cb689201",
            "source": "first-token-detail",
            "tokensPerSec": 700,
            "ttftMs": 17600
          },
          {
            "line": 830,
            "rx": "2026-10-28T14:19:26.937Z",
            "sessionId": "d6b02e4e-b6a0-465f-9b1a-bed90c610873",
            "source": "first-token-detail",
            "tokensPerSec": 594,
            "ttftMs": 13824
          },
          {
            "line": 1189,
            "rx": "2026-10-28T14:20:50.305Z",
            "sessionId": "cf562c5f-4e83-4755-8e51-1ad3f8adc512",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 17161
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-productivity-craftbook-research-to-document",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-research-to-document",
        "repetition": 1
      },
      "trialId": "craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-28T14-36-12-945Z-5kbv",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-28T14-36-12-945Z-5kbv/recording/chat-events.jsonl",
      "recordingSha256": "89c5047d5045f8ebc86528216036b40edb1e5fa72c341a8b70c3d53f6b60b55b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-28T14-36-12-945Z-5kbv/metrics.json",
      "metricsSha256": "0df1376b4f6b681133677f159bd0c543284e5790395fb739c4b6feadd46e5fbf",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 494,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-28T14:36:39.946Z",
            "sessionId": "e861b381-dcea-496d-b3ed-c95b5fe33bf6",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 20399
          },
          {
            "line": 191,
            "rx": "2026-10-28T14:37:19.969Z",
            "sessionId": "f67f3807-b133-4b78-90bc-619ce1cd7b88",
            "source": "first-token-detail",
            "tokensPerSec": 593,
            "ttftMs": 15389
          },
          {
            "line": 1153,
            "rx": "2026-10-28T14:40:21.762Z",
            "sessionId": "71e334c4-d69a-4d4e-bb48-05b081ca24b8",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 17094
          },
          {
            "line": 1456,
            "rx": "2026-10-28T14:42:15.160Z",
            "sessionId": "71e334c4-d69a-4d4e-bb48-05b081ca24b8",
            "source": "first-token-detail",
            "tokensPerSec": 448,
            "ttftMs": 59914
          },
          {
            "line": 1904,
            "rx": "2026-10-28T14:48:29.325Z",
            "sessionId": "00464ada-cb3e-418e-8216-e166b84626dd",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 53885
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-productivity-craftbook-research-to-document",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-research-to-document",
        "repetition": 1
      },
      "trialId": "craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-28T15-04-18-199Z-9l8o",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-28T15-04-18-199Z-9l8o/recording/chat-events.jsonl",
      "recordingSha256": "de9b8a1f6874a59540a26086113a9637fbbd99f42449348396feb1c41fb1f0f5",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-28T15-04-18-199Z-9l8o/metrics.json",
      "metricsSha256": "636f7682c7c5cbbb70ca03471a41cb6ea47095fc0ea290cc1efc9d3ce9c0f6ef",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1084,
        "sampleCount": 16,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 16,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T15:04:38.118Z",
            "sessionId": "c294b875-5837-4b7d-b089-e43f43fae37e",
            "source": "first-token-detail",
            "tokensPerSec": 1154,
            "ttftMs": 13403
          },
          {
            "line": 100,
            "rx": "2026-10-28T15:05:00.829Z",
            "sessionId": "532223e6-60a9-4404-b4da-4e8e73f6386d",
            "source": "first-token-detail",
            "tokensPerSec": 1483,
            "ttftMs": 6143
          },
          {
            "line": 224,
            "rx": "2026-10-28T15:05:31.525Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 1352,
            "ttftMs": 6785
          },
          {
            "line": 371,
            "rx": "2026-10-28T15:06:12.657Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 1204,
            "ttftMs": 12335
          },
          {
            "line": 944,
            "rx": "2026-10-28T15:09:15.488Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 967,
            "ttftMs": 21304
          },
          {
            "line": 2760,
            "rx": "2026-10-28T15:14:12.867Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 852,
            "ttftMs": 570
          },
          {
            "line": 3652,
            "rx": "2026-10-28T15:17:06.493Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 1082,
            "ttftMs": 22391
          },
          {
            "line": 4173,
            "rx": "2026-10-28T15:18:32.292Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 444,
            "ttftMs": 627
          },
          {
            "line": 4724,
            "rx": "2026-10-28T15:20:26.540Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 988,
            "ttftMs": 26330
          },
          {
            "line": 5173,
            "rx": "2026-10-28T15:21:38.893Z",
            "sessionId": "b53c9eda-f4f8-4853-8245-cde259f29a94",
            "source": "first-token-detail",
            "tokensPerSec": 420,
            "ttftMs": 642
          },
          {
            "line": 5597,
            "rx": "2026-10-28T15:23:07.452Z",
            "sessionId": "fc675020-4669-4af5-9ec7-915d59d6ed5a",
            "source": "first-token-detail",
            "tokensPerSec": 1191,
            "ttftMs": 11699
          },
          {
            "line": 5780,
            "rx": "2026-10-28T15:23:49.603Z",
            "sessionId": "ba30d901-656b-4f28-9e46-d2aec8f81cf7",
            "source": "first-token-detail",
            "tokensPerSec": 1086,
            "ttftMs": 9524
          },
          {
            "line": 6046,
            "rx": "2026-10-28T15:25:11.095Z",
            "sessionId": "ba30d901-656b-4f28-9e46-d2aec8f81cf7",
            "source": "first-token-detail",
            "tokensPerSec": 1120,
            "ttftMs": 15309
          },
          {
            "line": 7046,
            "rx": "2026-10-28T15:29:42.712Z",
            "sessionId": "ba30d901-656b-4f28-9e46-d2aec8f81cf7",
            "source": "first-token-detail",
            "tokensPerSec": 975,
            "ttftMs": 26205
          },
          {
            "line": 7582,
            "rx": "2026-10-28T15:31:12.431Z",
            "sessionId": "ba30d901-656b-4f28-9e46-d2aec8f81cf7",
            "source": "first-token-detail",
            "tokensPerSec": 827,
            "ttftMs": 898
          },
          {
            "line": 8911,
            "rx": "2026-10-28T15:40:42.607Z",
            "sessionId": "40112c10-0940-4c99-acb9-a7a6acd262a8",
            "source": "first-token-detail",
            "tokensPerSec": 1096,
            "ttftMs": 11568
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-fix-squisq-bugs",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "fix-squisq-bugs",
        "repetition": 1
      },
      "trialId": "fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-28T15-41-48-693Z-tr26",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-28T15-41-48-693Z-tr26/recording/chat-events.jsonl",
      "recordingSha256": "3e9cb8843e1cfab7146122fc60d688d3d554e297fcd7d47aae9603d2430b6b61",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-28T15-41-48-693Z-tr26/metrics.json",
      "metricsSha256": "eb617ee7232e2583bf0f8f55dc209748c6d3d8ac0e71affc94f8b1c5310e93e1",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 520,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T15:42:04.125Z",
            "sessionId": "f91ebbf7-ebdb-4cc8-9f18-1057d6992d17",
            "source": "first-token-detail",
            "tokensPerSec": 540,
            "ttftMs": 13755
          },
          {
            "line": 918,
            "rx": "2026-10-28T15:44:58.385Z",
            "sessionId": "f91ebbf7-ebdb-4cc8-9f18-1057d6992d17",
            "source": "first-token-detail",
            "tokensPerSec": 529,
            "ttftMs": 21793
          },
          {
            "line": 1116,
            "rx": "2026-10-28T15:46:19.039Z",
            "sessionId": "f91ebbf7-ebdb-4cc8-9f18-1057d6992d17",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 23054
          },
          {
            "line": 1498,
            "rx": "2026-10-28T15:47:27.828Z",
            "sessionId": "f91ebbf7-ebdb-4cc8-9f18-1057d6992d17",
            "source": "first-token-detail",
            "tokensPerSec": 460,
            "ttftMs": 1521
          },
          {
            "line": 1752,
            "rx": "2026-10-28T15:49:17.483Z",
            "sessionId": "f91ebbf7-ebdb-4cc8-9f18-1057d6992d17",
            "source": "first-token-detail",
            "tokensPerSec": 511,
            "ttftMs": 35238
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-fix-squisq-bugs",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "fix-squisq-bugs",
        "repetition": 1
      },
      "trialId": "fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-28T15-52-35-325Z-vkfu",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-28T15-52-35-325Z-vkfu/recording/chat-events.jsonl",
      "recordingSha256": "d70aa702fc21fca16de5d76e2dc5909fdcd862997c80d5827143d67452631025",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-28T15-52-35-325Z-vkfu/metrics.json",
      "metricsSha256": "18bc042ab2cb6907d07104612cda4002d1d46f0f38341c2ec3cc39c83ad5e9c1",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1013,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T15:52:48.048Z",
            "sessionId": "abf7d935-3cc2-4451-bd02-73c75a3cf2ec",
            "source": "first-token-detail",
            "tokensPerSec": 1013,
            "ttftMs": 11033
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-dev-craftbook-routing",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "dev-craftbook-routing",
        "repetition": 1
      },
      "trialId": "dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-28T15-58-57-938Z-0lmg",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-28T15-58-57-938Z-0lmg/recording/chat-events.jsonl",
      "recordingSha256": "cbb4321f4cbfded07934df339566c7fdadc3151edb0efb05287256dd3ae01418",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-28T15-58-57-938Z-0lmg/metrics.json",
      "metricsSha256": "eb29acaf9cae4e42b64e5d16112456c5d3c5c2a3db0d568cd4ac63bd23f06706",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 741.5,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T15:59:15.361Z",
            "sessionId": "0d24956e-0aaa-43c9-95b1-519912e693bb",
            "source": "first-token-detail",
            "tokensPerSec": 1077,
            "ttftMs": 15715
          },
          {
            "line": 1441,
            "rx": "2026-10-28T16:03:19.711Z",
            "sessionId": "0d24956e-0aaa-43c9-95b1-519912e693bb",
            "source": "first-token-detail",
            "tokensPerSec": 330,
            "ttftMs": 480
          },
          {
            "line": 1667,
            "rx": "2026-10-28T16:03:55.935Z",
            "sessionId": "0d24956e-0aaa-43c9-95b1-519912e693bb",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 676
          },
          {
            "line": 2018,
            "rx": "2026-10-28T16:04:54.819Z",
            "sessionId": "d77bcdfd-87a9-430c-b175-7baa108c2f3f",
            "source": "first-token-detail",
            "tokensPerSec": 927,
            "ttftMs": 7011
          },
          {
            "line": 2061,
            "rx": "2026-10-28T16:05:02.064Z",
            "sessionId": "0d24956e-0aaa-43c9-95b1-519912e693bb",
            "source": "first-token-detail",
            "tokensPerSec": 543,
            "ttftMs": 540
          },
          {
            "line": 2449,
            "rx": "2026-10-28T16:06:07.519Z",
            "sessionId": "d77bcdfd-87a9-430c-b175-7baa108c2f3f",
            "source": "first-token-detail",
            "tokensPerSec": 906,
            "ttftMs": 6134
          },
          {
            "line": 2483,
            "rx": "2026-10-28T16:06:14.714Z",
            "sessionId": "0d24956e-0aaa-43c9-95b1-519912e693bb",
            "source": "first-token-detail",
            "tokensPerSec": 577,
            "ttftMs": 678
          },
          {
            "line": 2562,
            "rx": "2026-10-28T16:06:26.560Z",
            "sessionId": "d77bcdfd-87a9-430c-b175-7baa108c2f3f",
            "source": "first-token-detail",
            "tokensPerSec": 552,
            "ttftMs": 936
          },
          {
            "line": 2602,
            "rx": "2026-10-28T16:06:45.549Z",
            "sessionId": "d77bcdfd-87a9-430c-b175-7baa108c2f3f",
            "source": "first-token-detail",
            "tokensPerSec": 1182,
            "ttftMs": 6876
          },
          {
            "line": 2646,
            "rx": "2026-10-28T16:07:09.720Z",
            "sessionId": "d77bcdfd-87a9-430c-b175-7baa108c2f3f",
            "source": "first-token-detail",
            "tokensPerSec": 1200,
            "ttftMs": 6350
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-dev-craftbook-routing",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "dev-craftbook-routing",
        "repetition": 1
      },
      "trialId": "dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-28T16-10-17-744Z-jvwm",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-28T16-10-17-744Z-jvwm/recording/chat-events.jsonl",
      "recordingSha256": "84f244646d1668a6d0f5457f00598ed11924dc32846154694224de1a7fde3309",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-28T16-10-17-744Z-jvwm/metrics.json",
      "metricsSha256": "6200396cd5030f963af60253aba0e89f7b1eb3a3f4ef9fb347b1d17fb8fb5dfc",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 524.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T16:10:46.580Z",
            "sessionId": "8a991aa2-703a-4cd8-87ce-1c8aed34c5a5",
            "source": "first-token-detail",
            "tokensPerSec": 576,
            "ttftMs": 27128
          },
          {
            "line": 2876,
            "rx": "2026-10-28T16:19:56.455Z",
            "sessionId": "22878a00-af3c-4dc9-8381-40c109c44b4b",
            "source": "first-token-detail",
            "tokensPerSec": 529,
            "ttftMs": 7998
          },
          {
            "line": 2952,
            "rx": "2026-10-28T16:20:14.221Z",
            "sessionId": "8a991aa2-703a-4cd8-87ce-1c8aed34c5a5",
            "source": "first-token-detail",
            "tokensPerSec": 236,
            "ttftMs": 1057
          },
          {
            "line": 3297,
            "rx": "2026-10-28T16:21:21.258Z",
            "sessionId": "288a0806-3624-43b3-8586-fb36f109088b",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 6966
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-interface-contract",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "interface-contract",
        "repetition": 1
      },
      "trialId": "interface-contract-mlx-gemma4-12b-q4-2026-10-28T16-25-00-876Z-0q67",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-gemma4-12b-q4-2026-10-28T16-25-00-876Z-0q67/recording/chat-events.jsonl",
      "recordingSha256": "909f1de866dcfbdf9b4bb5c7935ff24f37f950de79d29e907f559d7db0f6c9b4",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-gemma4-12b-q4-2026-10-28T16-25-00-876Z-0q67/metrics.json",
      "metricsSha256": "80095a87be59c0e63a9ba7e2fead80fdabd48a2b946a4cf4790fd1f68f704a65",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1148,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 24,
            "rx": "2026-10-28T16:25:18.378Z",
            "sessionId": "e05f87ac-10c6-43cf-960d-1e8dbd0671b7",
            "source": "first-token-detail",
            "tokensPerSec": 1078,
            "ttftMs": 15807
          },
          {
            "line": 169,
            "rx": "2026-10-28T16:25:48.457Z",
            "sessionId": "6d29373c-280a-44e9-b8bb-d6295e2b09ee",
            "source": "first-token-detail",
            "tokensPerSec": 1218,
            "ttftMs": 6072
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-interface-contract",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "interface-contract",
        "repetition": 1
      },
      "trialId": "interface-contract-mlx-qwen3-8-27b-q4-2026-10-28T16-27-50-316Z-2l0n",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-qwen3-8-27b-q4-2026-10-28T16-27-50-316Z-2l0n/recording/chat-events.jsonl",
      "recordingSha256": "9c0fe99b57d6946bbab3ad3bd4a2c5c2b3a83bd8bfead69d66f5cc536d45a708",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-qwen3-8-27b-q4-2026-10-28T16-27-50-316Z-2l0n/metrics.json",
      "metricsSha256": "48bbef7e83653fd9b085e8c3feeef6ed3791f7ef340c5201cca9ae8e3d79bd10",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 483.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T16:28:16.358Z",
            "sessionId": "03ca68d3-df23-4afe-9204-106eb95263ae",
            "source": "first-token-detail",
            "tokensPerSec": 674,
            "ttftMs": 24348
          },
          {
            "line": 174,
            "rx": "2026-10-28T16:28:38.077Z",
            "sessionId": "03ca68d3-df23-4afe-9204-106eb95263ae",
            "source": "first-token-detail",
            "tokensPerSec": 293,
            "ttftMs": 551
          },
          {
            "line": 201,
            "rx": "2026-10-28T16:28:41.487Z",
            "sessionId": "03ca68d3-df23-4afe-9204-106eb95263ae",
            "source": "first-token-detail",
            "tokensPerSec": 279,
            "ttftMs": 681
          },
          {
            "line": 270,
            "rx": "2026-10-28T16:29:00.874Z",
            "sessionId": "97d2351e-53d4-4371-a539-871d3271ab72",
            "source": "first-token-detail",
            "tokensPerSec": 700,
            "ttftMs": 9062
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-craftbook-code-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-code-review",
        "repetition": 1
      },
      "trialId": "craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-28T16-36-28-242Z-f2ak",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-28T16-36-28-242Z-f2ak/recording/chat-events.jsonl",
      "recordingSha256": "a8d33a56ad981d3e32f84a17276c7c55cda403b513dccabeead0515dbeb20ca0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-28T16-36-28-242Z-f2ak/metrics.json",
      "metricsSha256": "19b0a0a773a41a686b379ffcea735a60d069107691b18c41971aa708a1231903",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 599,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T16:36:53.813Z",
            "sessionId": "2bfe8c3e-0504-4704-b451-470dc9a76f0d",
            "source": "first-token-detail",
            "tokensPerSec": 580,
            "ttftMs": 18983
          },
          {
            "line": 373,
            "rx": "2026-10-28T16:38:13.290Z",
            "sessionId": "2bfe8c3e-0504-4704-b451-470dc9a76f0d",
            "source": "first-token-detail",
            "tokensPerSec": 618,
            "ttftMs": 18418
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-craftbook-code-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-code-review",
        "repetition": 1
      },
      "trialId": "craftbook-code-review-mlx-gemma4-12b-q4-2026-10-28T16-46-29-221Z-kxrz",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-gemma4-12b-q4-2026-10-28T16-46-29-221Z-kxrz/recording/chat-events.jsonl",
      "recordingSha256": "0f033f158b94daa71975dc6d33b03c35ed9bb5a71c456eb1a74838be3b5132b0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-gemma4-12b-q4-2026-10-28T16-46-29-221Z-kxrz/metrics.json",
      "metricsSha256": "e2d968a0358784e0a77b9c5f3dc7dc88f1f8cb96dc78b5275259ceb22fea7785",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1220,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T16:46:49.064Z",
            "sessionId": "2b52afdc-0740-4901-a973-f8a4a91921e1",
            "source": "first-token-detail",
            "tokensPerSec": 1085,
            "ttftMs": 13294
          },
          {
            "line": 112,
            "rx": "2026-10-28T16:47:15.007Z",
            "sessionId": "2b52afdc-0740-4901-a973-f8a4a91921e1",
            "source": "first-token-detail",
            "tokensPerSec": 1160,
            "ttftMs": 9271
          },
          {
            "line": 354,
            "rx": "2026-10-28T16:48:08.518Z",
            "sessionId": "2b52afdc-0740-4901-a973-f8a4a91921e1",
            "source": "first-token-detail",
            "tokensPerSec": 1280,
            "ttftMs": 7681
          },
          {
            "line": 432,
            "rx": "2026-10-28T16:48:30.221Z",
            "sessionId": "7aa4d9e0-ff67-4cd1-99d9-a6545075cc5f",
            "source": "first-token-detail",
            "tokensPerSec": 1658,
            "ttftMs": 3890
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-craftbook-api-contract-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-api-contract-review",
        "repetition": 1
      },
      "trialId": "craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-28T16-48-59-569Z-uudw",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-28T16-48-59-569Z-uudw/recording/chat-events.jsonl",
      "recordingSha256": "880f4e83382866aef02b20cac6a33b503d90b479e3c6998a334f0f1874d8e16c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-28T16-48-59-569Z-uudw/metrics.json",
      "metricsSha256": "11e2200a90599cf80493d6632ad88a3f57b814eb8bee1f8f18c5c8741227eac0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 538,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-28T16:49:26.887Z",
            "sessionId": "18cb426a-3b73-425f-be3e-8edc6abeee68",
            "source": "first-token-detail",
            "tokensPerSec": 600,
            "ttftMs": 20776
          },
          {
            "line": 1656,
            "rx": "2026-10-28T16:54:21.237Z",
            "sessionId": "25cf91b1-2eaf-4537-9a28-d7b50bc06c20",
            "source": "first-token-detail",
            "tokensPerSec": 528,
            "ttftMs": 14955
          },
          {
            "line": 3045,
            "rx": "2026-10-28T16:58:59.125Z",
            "sessionId": "60ec20fc-c78d-4418-bf74-10fe2c659f59",
            "source": "first-token-detail",
            "tokensPerSec": 548,
            "ttftMs": 7789
          },
          {
            "line": 5531,
            "rx": "2026-10-28T17:06:57.655Z",
            "sessionId": "60ec20fc-c78d-4418-bf74-10fe2c659f59",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 26231
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-craftbook-api-contract-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-api-contract-review",
        "repetition": 1
      },
      "trialId": "craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-28T17-24-25-881Z-vzmh",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-28T17-24-25-881Z-vzmh/recording/chat-events.jsonl",
      "recordingSha256": "7e98dbf86a7fa7013111219bc1495fb55f99ad2ff13fe3dc1d357248871672d0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-28T17-24-25-881Z-vzmh/metrics.json",
      "metricsSha256": "86ae9419412347c55489ddccf7e727be358c300a886cc7c9270e8f36f8e30fd0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1085,
        "sampleCount": 8,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 8,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T17:24:47.201Z",
            "sessionId": "728e12da-b953-4853-aa56-fbe7fbd86454",
            "source": "first-token-detail",
            "tokensPerSec": 1037,
            "ttftMs": 14786
          },
          {
            "line": 1721,
            "rx": "2026-10-28T17:29:58.892Z",
            "sessionId": "2a25ae79-a47d-4f75-89e3-0b538302dc06",
            "source": "first-token-detail",
            "tokensPerSec": 1303,
            "ttftMs": 6350
          },
          {
            "line": 1890,
            "rx": "2026-10-28T17:30:48.078Z",
            "sessionId": "c70389b2-34c7-42f2-bf88-800613e54d51",
            "source": "first-token-detail",
            "tokensPerSec": 1257,
            "ttftMs": 5531
          },
          {
            "line": 2112,
            "rx": "2026-10-28T17:31:41.723Z",
            "sessionId": "c70389b2-34c7-42f2-bf88-800613e54d51",
            "source": "first-token-detail",
            "tokensPerSec": 1070,
            "ttftMs": 9229
          },
          {
            "line": 2672,
            "rx": "2026-10-28T17:33:11.406Z",
            "sessionId": "c70389b2-34c7-42f2-bf88-800613e54d51",
            "source": "first-token-detail",
            "tokensPerSec": 795,
            "ttftMs": 812
          },
          {
            "line": 3214,
            "rx": "2026-10-28T17:34:37.129Z",
            "sessionId": "c70389b2-34c7-42f2-bf88-800613e54d51",
            "source": "first-token-detail",
            "tokensPerSec": 736,
            "ttftMs": 872
          },
          {
            "line": 3431,
            "rx": "2026-10-28T17:35:24.533Z",
            "sessionId": "a4303158-6241-4885-9415-95514bf82449",
            "source": "first-token-detail",
            "tokensPerSec": 1100,
            "ttftMs": 6938
          },
          {
            "line": 3551,
            "rx": "2026-10-28T17:35:49.264Z",
            "sessionId": "706e654c-001a-4b46-a9ad-30a91a2fb696",
            "source": "first-token-detail",
            "tokensPerSec": 1449,
            "ttftMs": 3350
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-craftbook-deep-security-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-deep-security-review",
        "repetition": 1
      },
      "trialId": "craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-28T17-50-41-916Z-2i20",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-28T17-50-41-916Z-2i20/recording/chat-events.jsonl",
      "recordingSha256": "0c8c829db7fba2d4d25cbf0c1c35673ccc2d2488ab3c3e0598f1a3b36b87d060",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-28T17-50-41-916Z-2i20/metrics.json",
      "metricsSha256": "4352c097dcb7e39ab9e67dbe0a19a96817b5edce45b653ee4fac51dec72e6a7e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 321,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-28T17:50:57.522Z",
            "sessionId": "549bf309-ec13-4984-bcc1-db145b926cbd",
            "source": "first-token-detail",
            "tokensPerSec": 1084,
            "ttftMs": 13746
          },
          {
            "line": 1375,
            "rx": "2026-10-28T17:54:40.840Z",
            "sessionId": "549bf309-ec13-4984-bcc1-db145b926cbd",
            "source": "first-token-detail",
            "tokensPerSec": 321,
            "ttftMs": 1677
          },
          {
            "line": 1655,
            "rx": "2026-10-28T17:55:36.144Z",
            "sessionId": "6f50335d-bf44-4152-a1d6-d42d6d05cc4a",
            "source": "first-token-detail",
            "tokensPerSec": 98,
            "ttftMs": 56992
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-craftbook-deep-security-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-deep-security-review",
        "repetition": 1
      },
      "trialId": "craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-28T18-02-49-769Z-hfjl",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-28T18-02-49-769Z-hfjl/recording/chat-events.jsonl",
      "recordingSha256": "b343e9a5b927e9bd24fb69d648cb08d6adf37dc9aae4a199ee040e9b535cc148",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-28T18-02-49-769Z-hfjl/metrics.json",
      "metricsSha256": "256adf3562defaaa8a587a7c8c7521afa05f5f2f6b6d055f2b8f679296bcc77d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 562,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T18:03:13.517Z",
            "sessionId": "ab68f916-8095-4758-b951-c9c56abe43b6",
            "source": "first-token-detail",
            "tokensPerSec": 555,
            "ttftMs": 22067
          },
          {
            "line": 1655,
            "rx": "2026-10-28T18:07:55.968Z",
            "sessionId": "f6fc9940-2f00-4f4d-8072-cc63088255da",
            "source": "first-token-detail",
            "tokensPerSec": 508,
            "ttftMs": 9412
          },
          {
            "line": 1905,
            "rx": "2026-10-28T18:08:58.786Z",
            "sessionId": "f6fc9940-2f00-4f4d-8072-cc63088255da",
            "source": "first-token-detail",
            "tokensPerSec": 562,
            "ttftMs": 13650
          },
          {
            "line": 1971,
            "rx": "2026-10-28T18:09:35.531Z",
            "sessionId": "f6fc9940-2f00-4f4d-8072-cc63088255da",
            "source": "first-token-detail",
            "tokensPerSec": 603,
            "ttftMs": 10484
          },
          {
            "line": 2031,
            "rx": "2026-10-28T18:09:57.655Z",
            "sessionId": "f6fc9940-2f00-4f4d-8072-cc63088255da",
            "source": "first-token-detail",
            "tokensPerSec": 567,
            "ttftMs": 11895
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-large-pr-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "large-pr-review",
        "repetition": 1
      },
      "trialId": "large-pr-review-mlx-gemma4-12b-q4-2026-10-28T18-10-52-383Z-zc7k",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-gemma4-12b-q4-2026-10-28T18-10-52-383Z-zc7k/recording/chat-events.jsonl",
      "recordingSha256": "ed9d8990153c183a6f0e8d8d81317e788bfd0fe76556c7edbaceb47f83269d28",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-gemma4-12b-q4-2026-10-28T18-10-52-383Z-zc7k/metrics.json",
      "metricsSha256": "d618c050bdb638b7e48574612b24bbda9be5d51bdbe50e3f3a2df2c20d0e2f2e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 361,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-28T18:11:10.760Z",
            "sessionId": "95c17ad2-f447-4501-ba23-38aab1104958",
            "source": "first-token-detail",
            "tokensPerSec": 1146,
            "ttftMs": 16605
          },
          {
            "line": 1380,
            "rx": "2026-10-28T18:14:57.362Z",
            "sessionId": "95c17ad2-f447-4501-ba23-38aab1104958",
            "source": "first-token-detail",
            "tokensPerSec": 478,
            "ttftMs": 545
          },
          {
            "line": 2332,
            "rx": "2026-10-28T18:22:13.521Z",
            "sessionId": "95c17ad2-f447-4501-ba23-38aab1104958",
            "source": "first-token-detail",
            "tokensPerSec": 361,
            "ttftMs": 3232
          },
          {
            "line": 5493,
            "rx": "2026-10-28T18:39:10.736Z",
            "sessionId": "95c17ad2-f447-4501-ba23-38aab1104958",
            "source": "first-token-detail",
            "tokensPerSec": 289,
            "ttftMs": 3597
          },
          {
            "line": 8834,
            "rx": "2026-10-28T18:57:39.416Z",
            "sessionId": "95c17ad2-f447-4501-ba23-38aab1104958",
            "source": "first-token-detail",
            "tokensPerSec": 174,
            "ttftMs": 1488
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-large-pr-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "large-pr-review",
        "repetition": 1
      },
      "trialId": "large-pr-review-mlx-qwen3-8-27b-q4-2026-10-28T19-13-05-081Z-wfxq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-qwen3-8-27b-q4-2026-10-28T19-13-05-081Z-wfxq/recording/chat-events.jsonl",
      "recordingSha256": "e485326c2be722125fad7e30145e35eeb40c13cfa59f82ac445e53fa81321cf4",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-qwen3-8-27b-q4-2026-10-28T19-13-05-081Z-wfxq/metrics.json",
      "metricsSha256": "7bfc48e00995e698c79ae27fc3e0cf92a9d4d464e8ede5322d2ff73b78a49d4a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 389,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-28T19:13:36.911Z",
            "sessionId": "2c4bbe28-310f-497c-9f52-2d06efe3dcf6",
            "source": "first-token-detail",
            "tokensPerSec": 603,
            "ttftMs": 30028
          },
          {
            "line": 10104,
            "rx": "2026-10-28T19:48:27.628Z",
            "sessionId": "2c4bbe28-310f-497c-9f52-2d06efe3dcf6",
            "source": "first-token-detail",
            "tokensPerSec": 175,
            "ttftMs": 4503
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-craftbook-codemod-sweep",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-codemod-sweep",
        "repetition": 1
      },
      "trialId": "craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-28T20-54-52-653Z-7pru",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-28T20-54-52-653Z-7pru/recording/chat-events.jsonl",
      "recordingSha256": "60f7706625c198c7b04940b102fa9afe6586f6a933faa6adb3d3a5a3b21b92ee",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-28T20-54-52-653Z-7pru/metrics.json",
      "metricsSha256": "606e82b0db60c8bbf349676bbb6f24940f6af21919a3e2c9cb154bec09e2a883",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 468,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T20:55:15.953Z",
            "sessionId": "40b83b7c-69d1-46bd-af87-2ca67bb6882b",
            "source": "first-token-detail",
            "tokensPerSec": 835,
            "ttftMs": 16769
          },
          {
            "line": 1016,
            "rx": "2026-10-28T20:58:28.038Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 374,
            "ttftMs": 23728
          },
          {
            "line": 2192,
            "rx": "2026-10-28T21:02:19.916Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 486,
            "ttftMs": 20508
          },
          {
            "line": 3028,
            "rx": "2026-10-28T21:04:54.497Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 214,
            "ttftMs": 1036
          },
          {
            "line": 3244,
            "rx": "2026-10-28T21:05:54.595Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 489,
            "ttftMs": 23169
          },
          {
            "line": 4049,
            "rx": "2026-10-28T21:08:22.519Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 474,
            "ttftMs": 1295
          },
          {
            "line": 4764,
            "rx": "2026-10-28T21:10:41.711Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 367,
            "ttftMs": 973
          },
          {
            "line": 5249,
            "rx": "2026-10-28T21:12:10.489Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 468,
            "ttftMs": 1290
          },
          {
            "line": 7698,
            "rx": "2026-10-28T21:19:19.407Z",
            "sessionId": "c79b1653-4d72-400f-9b3c-335eed55f13b",
            "source": "first-token-detail",
            "tokensPerSec": 275,
            "ttftMs": 1507
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-craftbook-codemod-sweep",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-codemod-sweep",
        "repetition": 1
      },
      "trialId": "craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-28T22-58-54-564Z-dp8j",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-28T22-58-54-564Z-dp8j/recording/chat-events.jsonl",
      "recordingSha256": "4663880eaa9813b64fd54bd4927f875407c3c584b860078415ef667a935b7e12",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-28T22-58-54-564Z-dp8j/metrics.json",
      "metricsSha256": "09ac897ea0428340585a98399251183cd1b1917316b9e09565b6f333bface92e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 818.5,
        "sampleCount": 18,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 18,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-28T22:59:16.232Z",
            "sessionId": "d8ae4f3c-e1be-4156-b4fc-4ea1412ef5b9",
            "source": "first-token-detail",
            "tokensPerSec": 936,
            "ttftMs": 15147
          },
          {
            "line": 158,
            "rx": "2026-10-28T23:00:03.253Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 1297,
            "ttftMs": 7146
          },
          {
            "line": 281,
            "rx": "2026-10-28T23:00:36.468Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 1088,
            "ttftMs": 9462
          },
          {
            "line": 621,
            "rx": "2026-10-28T23:01:40.966Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 1092,
            "ttftMs": 9800
          },
          {
            "line": 867,
            "rx": "2026-10-28T23:02:22.708Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 514,
            "ttftMs": 740
          },
          {
            "line": 1296,
            "rx": "2026-10-28T23:03:34.037Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 524,
            "ttftMs": 821
          },
          {
            "line": 3568,
            "rx": "2026-10-28T23:09:53.755Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 420,
            "ttftMs": 1017
          },
          {
            "line": 4096,
            "rx": "2026-10-28T23:11:39.008Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 898,
            "ttftMs": 24043
          },
          {
            "line": 4512,
            "rx": "2026-10-28T23:12:45.689Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 813,
            "ttftMs": 818
          },
          {
            "line": 4915,
            "rx": "2026-10-28T23:14:12.334Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 890,
            "ttftMs": 25837
          },
          {
            "line": 5442,
            "rx": "2026-10-28T23:16:13.334Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 831,
            "ttftMs": 29529
          },
          {
            "line": 5928,
            "rx": "2026-10-28T23:18:00.515Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 847,
            "ttftMs": 28466
          },
          {
            "line": 6296,
            "rx": "2026-10-28T23:19:35.458Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 791,
            "ttftMs": 32994
          },
          {
            "line": 6339,
            "rx": "2026-10-28T23:19:47.868Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 448,
            "ttftMs": 999
          },
          {
            "line": 6561,
            "rx": "2026-10-28T23:21:00.759Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 824,
            "ttftMs": 34717
          },
          {
            "line": 6970,
            "rx": "2026-10-28T23:22:11.056Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 676,
            "ttftMs": 611
          },
          {
            "line": 7443,
            "rx": "2026-10-28T23:23:30.469Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 689,
            "ttftMs": 528
          },
          {
            "line": 7723,
            "rx": "2026-10-28T23:24:17.776Z",
            "sessionId": "2ccf4599-4395-44c5-a8ab-049dc8b37656",
            "source": "first-token-detail",
            "tokensPerSec": 743,
            "ttftMs": 744
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-craftbook-refactor-module",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-refactor-module",
        "repetition": 1
      },
      "trialId": "craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-28T23-26-52-344Z-bssk",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-28T23-26-52-344Z-bssk/recording/chat-events.jsonl",
      "recordingSha256": "9910bcce22ba8028629b4e0a8799df5826d81a249f6294fbf0ae3bf24089db9b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-28T23-26-52-344Z-bssk/metrics.json",
      "metricsSha256": "0d07a023bc231ac37a140565e3843f586dd428c1d784efad16151cfc67fb0832",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 485.5,
        "sampleCount": 22,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 22,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-28T23:27:14.929Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 548,
            "ttftMs": 16057
          },
          {
            "line": 149,
            "rx": "2026-10-28T23:27:35.298Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 479,
            "ttftMs": 1949
          },
          {
            "line": 289,
            "rx": "2026-10-28T23:27:58.910Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 488,
            "ttftMs": 1911
          },
          {
            "line": 459,
            "rx": "2026-10-28T23:28:45.993Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 21610
          },
          {
            "line": 2565,
            "rx": "2026-10-28T23:34:51.153Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 238,
            "ttftMs": 1118
          },
          {
            "line": 2923,
            "rx": "2026-10-28T23:35:54.428Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 216,
            "ttftMs": 1103
          },
          {
            "line": 3692,
            "rx": "2026-10-28T23:38:52.469Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 483,
            "ttftMs": 35500
          },
          {
            "line": 4358,
            "rx": "2026-10-28T23:41:25.961Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 473,
            "ttftMs": 36680
          },
          {
            "line": 4497,
            "rx": "2026-10-28T23:41:50.671Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 442,
            "ttftMs": 1233
          },
          {
            "line": 4603,
            "rx": "2026-10-28T23:42:10.665Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 346,
            "ttftMs": 945
          },
          {
            "line": 4747,
            "rx": "2026-10-28T23:43:43.908Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 33418
          },
          {
            "line": 4857,
            "rx": "2026-10-28T23:46:31.064Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 755,
            "ttftMs": 23140
          },
          {
            "line": 4979,
            "rx": "2026-10-28T23:46:49.629Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 489,
            "ttftMs": 1208
          },
          {
            "line": 5158,
            "rx": "2026-10-28T23:50:15.351Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 657,
            "ttftMs": 29387
          },
          {
            "line": 5280,
            "rx": "2026-10-28T23:53:35.711Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 554,
            "ttftMs": 1330
          },
          {
            "line": 5401,
            "rx": "2026-10-28T23:54:56.797Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 1170
          },
          {
            "line": 5508,
            "rx": "2026-10-28T23:56:17.982Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 469,
            "ttftMs": 1105
          },
          {
            "line": 5543,
            "rx": "2026-10-28T23:57:24.774Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 479,
            "ttftMs": 1011
          },
          {
            "line": 5656,
            "rx": "2026-10-28T23:58:47.162Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 1091
          },
          {
            "line": 5763,
            "rx": "2026-10-29T00:00:09.355Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 459,
            "ttftMs": 1019
          },
          {
            "line": 5868,
            "rx": "2026-10-29T00:01:34.829Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 573,
            "ttftMs": 40245
          },
          {
            "line": 5952,
            "rx": "2026-10-29T00:03:25.998Z",
            "sessionId": "042f5a66-3adb-449f-b4eb-7edb6f8fc320",
            "source": "first-token-detail",
            "tokensPerSec": 418,
            "ttftMs": 58819
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-craftbook-refactor-module",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-refactor-module",
        "repetition": 1
      },
      "trialId": "craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-29T00-45-48-041Z-7jsj",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-29T00-45-48-041Z-7jsj/recording/chat-events.jsonl",
      "recordingSha256": "98dbaa66018b585a014fd10641c4c428104a2ebd935345bc72de2f25c75c4f53",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-29T00-45-48-041Z-7jsj/metrics.json",
      "metricsSha256": "6afb5d809f4374d59a651bd1dbd77eb636f1605316ffdb340a3bf7e2dddad930",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1101.5,
        "sampleCount": 24,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 24,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-29T00:46:06.490Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1145,
            "ttftMs": 11910
          },
          {
            "line": 71,
            "rx": "2026-10-29T00:46:17.912Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 966,
            "ttftMs": 1387
          },
          {
            "line": 111,
            "rx": "2026-10-29T00:46:28.990Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 952,
            "ttftMs": 1407
          },
          {
            "line": 164,
            "rx": "2026-10-29T00:46:50.989Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1206,
            "ttftMs": 11155
          },
          {
            "line": 295,
            "rx": "2026-10-29T00:47:15.094Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 689,
            "ttftMs": 898
          },
          {
            "line": 518,
            "rx": "2026-10-29T00:48:03.020Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1200,
            "ttftMs": 12789
          },
          {
            "line": 826,
            "rx": "2026-10-29T00:48:57.372Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 591,
            "ttftMs": 1008
          },
          {
            "line": 1246,
            "rx": "2026-10-29T00:50:25.564Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 966,
            "ttftMs": 17088
          },
          {
            "line": 1288,
            "rx": "2026-10-29T00:50:39.244Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1114,
            "ttftMs": 835
          },
          {
            "line": 1306,
            "rx": "2026-10-29T00:50:44.077Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 727,
            "ttftMs": 592
          },
          {
            "line": 1347,
            "rx": "2026-10-29T00:51:26.799Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 946,
            "ttftMs": 17988
          },
          {
            "line": 1411,
            "rx": "2026-10-29T00:51:57.930Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 921,
            "ttftMs": 21151
          },
          {
            "line": 2237,
            "rx": "2026-10-29T00:54:26.708Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 890,
            "ttftMs": 20177
          },
          {
            "line": 2278,
            "rx": "2026-10-29T00:54:38.442Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 654,
            "ttftMs": 1440
          },
          {
            "line": 2323,
            "rx": "2026-10-29T00:57:47.084Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1601,
            "ttftMs": 12446
          },
          {
            "line": 2358,
            "rx": "2026-10-29T01:00:53.576Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1288,
            "ttftMs": 901
          },
          {
            "line": 2386,
            "rx": "2026-10-29T01:02:04.539Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1145,
            "ttftMs": 772
          },
          {
            "line": 2423,
            "rx": "2026-10-29T01:03:15.656Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 784
          },
          {
            "line": 2460,
            "rx": "2026-10-29T01:03:25.864Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1348,
            "ttftMs": 836
          },
          {
            "line": 2497,
            "rx": "2026-10-29T01:04:10.397Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1230,
            "ttftMs": 19982
          },
          {
            "line": 2530,
            "rx": "2026-10-29T01:05:22.602Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1165,
            "ttftMs": 830
          },
          {
            "line": 2554,
            "rx": "2026-10-29T01:06:28.475Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1059,
            "ttftMs": 712
          },
          {
            "line": 2578,
            "rx": "2026-10-29T01:06:33.621Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1234,
            "ttftMs": 776
          },
          {
            "line": 2602,
            "rx": "2026-10-29T01:06:38.810Z",
            "sessionId": "cebc8916-1769-4c84-8c4f-b8646ad36065",
            "source": "first-token-detail",
            "tokensPerSec": 1089,
            "ttftMs": 876
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-developer-craftbook-bug-fix-tdd",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-bug-fix-tdd",
        "repetition": 1
      },
      "trialId": "craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-29T02-39-20-666Z-fe1m",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-29T02-39-20-666Z-fe1m/recording/chat-events.jsonl",
      "recordingSha256": "eec80cd5b64e4a93b9091ecd1f253ca31267c7927f43aabbb1c777ffb98991b2",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-29T02-39-20-666Z-fe1m/metrics.json",
      "metricsSha256": "f97b9ea8683bd4e57949281de19d3775fd4b27bfbe6afad1e457eedca9483d56",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 906.5,
        "sampleCount": 42,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 42,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-29T02:39:42.516Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1050,
            "ttftMs": 15311
          },
          {
            "line": 213,
            "rx": "2026-10-29T02:40:30.498Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1067,
            "ttftMs": 9882
          },
          {
            "line": 240,
            "rx": "2026-10-29T02:40:37.082Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 907,
            "ttftMs": 1145
          },
          {
            "line": 272,
            "rx": "2026-10-29T02:40:44.641Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 851,
            "ttftMs": 961
          },
          {
            "line": 299,
            "rx": "2026-10-29T02:40:59.993Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1037,
            "ttftMs": 12425
          },
          {
            "line": 709,
            "rx": "2026-10-29T02:42:21.050Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1168,
            "ttftMs": 13011
          },
          {
            "line": 948,
            "rx": "2026-10-29T02:43:01.802Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 612,
            "ttftMs": 763
          },
          {
            "line": 1359,
            "rx": "2026-10-29T02:44:08.926Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 538,
            "ttftMs": 866
          },
          {
            "line": 1529,
            "rx": "2026-10-29T02:44:54.407Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 974,
            "ttftMs": 15285
          },
          {
            "line": 1562,
            "rx": "2026-10-29T02:45:04.799Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1182,
            "ttftMs": 734
          },
          {
            "line": 1589,
            "rx": "2026-10-29T02:45:28.022Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1008,
            "ttftMs": 18399
          },
          {
            "line": 1832,
            "rx": "2026-10-29T02:46:27.750Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 945,
            "ttftMs": 20378
          },
          {
            "line": 2138,
            "rx": "2026-10-29T02:47:35.627Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1016,
            "ttftMs": 18249
          },
          {
            "line": 2169,
            "rx": "2026-10-29T02:47:48.185Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1102,
            "ttftMs": 756
          },
          {
            "line": 2196,
            "rx": "2026-10-29T02:48:13.270Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1013,
            "ttftMs": 20268
          },
          {
            "line": 2553,
            "rx": "2026-10-29T02:49:32.404Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 930,
            "ttftMs": 22828
          },
          {
            "line": 2593,
            "rx": "2026-10-29T02:50:06.175Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 972,
            "ttftMs": 23813
          },
          {
            "line": 2862,
            "rx": "2026-10-29T02:51:16.816Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 942,
            "ttftMs": 24920
          },
          {
            "line": 2903,
            "rx": "2026-10-29T02:51:50.360Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 913,
            "ttftMs": 27214
          },
          {
            "line": 3067,
            "rx": "2026-10-29T02:52:45.177Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 867,
            "ttftMs": 29570
          },
          {
            "line": 3120,
            "rx": "2026-10-29T02:53:30.782Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 830,
            "ttftMs": 33297
          },
          {
            "line": 3289,
            "rx": "2026-10-29T02:54:25.727Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 906,
            "ttftMs": 29715
          },
          {
            "line": 3317,
            "rx": "2026-10-29T02:54:32.785Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 891,
            "ttftMs": 749
          },
          {
            "line": 3371,
            "rx": "2026-10-29T02:55:16.461Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 870,
            "ttftMs": 33424
          },
          {
            "line": 3621,
            "rx": "2026-10-29T02:56:30.887Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 903,
            "ttftMs": 33621
          },
          {
            "line": 3669,
            "rx": "2026-10-29T02:57:14.093Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 875,
            "ttftMs": 36377
          },
          {
            "line": 3852,
            "rx": "2026-10-29T02:58:16.285Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 901,
            "ttftMs": 34360
          },
          {
            "line": 3890,
            "rx": "2026-10-29T02:58:29.609Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1017,
            "ttftMs": 1045
          },
          {
            "line": 3929,
            "rx": "2026-10-29T02:58:40.641Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 586,
            "ttftMs": 1550
          },
          {
            "line": 3977,
            "rx": "2026-10-29T02:59:28.125Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 899,
            "ttftMs": 37934
          },
          {
            "line": 4123,
            "rx": "2026-10-29T03:00:33.144Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 880,
            "ttftMs": 40120
          },
          {
            "line": 4151,
            "rx": "2026-10-29T03:00:40.814Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 598,
            "ttftMs": 1330
          },
          {
            "line": 4199,
            "rx": "2026-10-29T03:01:33.434Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 873,
            "ttftMs": 42890
          },
          {
            "line": 4404,
            "rx": "2026-10-29T03:02:53.643Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 867,
            "ttftMs": 44468
          },
          {
            "line": 4445,
            "rx": "2026-10-29T03:03:07.239Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 872,
            "ttftMs": 1098
          },
          {
            "line": 4495,
            "rx": "2026-10-29T03:04:05.217Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 849,
            "ttftMs": 48018
          },
          {
            "line": 4993,
            "rx": "2026-10-29T03:06:13.434Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 838,
            "ttftMs": 50043
          },
          {
            "line": 5020,
            "rx": "2026-10-29T03:06:21.359Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 804,
            "ttftMs": 1016
          },
          {
            "line": 5041,
            "rx": "2026-10-29T03:09:26.754Z",
            "sessionId": "one-shot:775efb08-3230-4dc6-9572-baad3d3f0b1b",
            "source": "first-token-detail",
            "tokensPerSec": 958,
            "ttftMs": 1204
          },
          {
            "line": 5072,
            "rx": "2026-10-29T03:11:22.488Z",
            "sessionId": "one-shot:1347ca8e-db41-4a0d-b74b-cbb202c19be5",
            "source": "first-token-detail",
            "tokensPerSec": 1576,
            "ttftMs": 745
          },
          {
            "line": 5160,
            "rx": "2026-10-29T03:15:00.968Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1284,
            "ttftMs": 34577
          },
          {
            "line": 5431,
            "rx": "2026-10-29T03:16:25.481Z",
            "sessionId": "03b14ccd-4336-47b6-a1b5-8cb88bc3fe8b",
            "source": "first-token-detail",
            "tokensPerSec": 1108,
            "ttftMs": 40455
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-developer-craftbook-bug-fix-tdd",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-bug-fix-tdd",
        "repetition": 1
      },
      "trialId": "craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-29T03-49-52-142Z-n6ra",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-29T03-49-52-142Z-n6ra/recording/chat-events.jsonl",
      "recordingSha256": "5f97d4d0c1fce8633d1cc8c4b8f82520b8409b53497d984a844242ccafe37312",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-29T03-49-52-142Z-n6ra/metrics.json",
      "metricsSha256": "052c51c9412f4c8ee11b6eccc8a7e45271c76ae5c6b56b9f6292290ed85c0f50",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 502,
        "sampleCount": 23,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 23,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-29T03:50:20.449Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 540,
            "ttftMs": 21782
          },
          {
            "line": 1734,
            "rx": "2026-10-29T03:55:07.612Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 235,
            "ttftMs": 1014
          },
          {
            "line": 2810,
            "rx": "2026-10-29T03:58:22.907Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 271,
            "ttftMs": 1130
          },
          {
            "line": 3195,
            "rx": "2026-10-29T03:59:49.143Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 19772
          },
          {
            "line": 3275,
            "rx": "2026-10-29T04:00:33.962Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 16430
          },
          {
            "line": 3337,
            "rx": "2026-10-29T04:01:25.487Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 29302
          },
          {
            "line": 3882,
            "rx": "2026-10-29T04:03:04.648Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 224,
            "ttftMs": 1060
          },
          {
            "line": 4399,
            "rx": "2026-10-29T04:05:00.607Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 524,
            "ttftMs": 20729
          },
          {
            "line": 4482,
            "rx": "2026-10-29T04:05:15.901Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 931
          },
          {
            "line": 4642,
            "rx": "2026-10-29T04:05:46.284Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 445,
            "ttftMs": 716
          },
          {
            "line": 4727,
            "rx": "2026-10-29T04:06:47.138Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 25788
          },
          {
            "line": 4837,
            "rx": "2026-10-29T04:09:42.475Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 772,
            "ttftMs": 18224
          },
          {
            "line": 4925,
            "rx": "2026-10-29T04:09:56.694Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 578,
            "ttftMs": 1098
          },
          {
            "line": 4995,
            "rx": "2026-10-29T04:13:20.395Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 751,
            "ttftMs": 21830
          },
          {
            "line": 5088,
            "rx": "2026-10-29T04:16:39.271Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 560,
            "ttftMs": 1292
          },
          {
            "line": 5198,
            "rx": "2026-10-29T04:19:57.246Z",
            "sessionId": "one-shot:56491f7e-4f1e-4de9-92b3-10b4cb089b08",
            "source": "first-token-detail",
            "tokensPerSec": 708,
            "ttftMs": 1313
          },
          {
            "line": 5213,
            "rx": "2026-10-29T04:20:02.951Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 1373
          },
          {
            "line": 5340,
            "rx": "2026-10-29T04:21:26.296Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 498,
            "ttftMs": 1078
          },
          {
            "line": 5437,
            "rx": "2026-10-29T04:22:44.769Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 1204
          },
          {
            "line": 5550,
            "rx": "2026-10-29T04:24:08.268Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 484,
            "ttftMs": 1089
          },
          {
            "line": 5654,
            "rx": "2026-10-29T04:25:31.804Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 461,
            "ttftMs": 1013
          },
          {
            "line": 5719,
            "rx": "2026-10-29T04:26:44.463Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 574,
            "ttftMs": 37016
          },
          {
            "line": 5807,
            "rx": "2026-10-29T04:28:17.053Z",
            "sessionId": "5bd8af2c-82b9-4332-be4f-62344af060d5",
            "source": "first-token-detail",
            "tokensPerSec": 503,
            "ttftMs": 45717
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-find-vs-create",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-find-vs-create",
        "repetition": 1
      },
      "trialId": "craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-29T04-31-56-174Z-jchw",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-29T04-31-56-174Z-jchw/recording/chat-events.jsonl",
      "recordingSha256": "1c532f549bf990d1bf9981492840484ee68f4769f7a58963cf6b2779a8d654bf",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-29T04-31-56-174Z-jchw/metrics.json",
      "metricsSha256": "8753b8fd748a6a738249003acfa3da3898ae61f899e00e0ad742314598306475",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 982,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-29T04:32:14.763Z",
            "sessionId": "655645fa-0f9e-4949-9be3-9905a16d75d9",
            "source": "first-token-detail",
            "tokensPerSec": 958,
            "ttftMs": 16877
          },
          {
            "line": 1585,
            "rx": "2026-10-29T04:36:47.527Z",
            "sessionId": "655645fa-0f9e-4949-9be3-9905a16d75d9",
            "source": "first-token-detail",
            "tokensPerSec": 255,
            "ttftMs": 433
          },
          {
            "line": 1976,
            "rx": "2026-10-29T04:37:54.634Z",
            "sessionId": "044053b2-d379-4cdb-b2c3-df1e81a850b4",
            "source": "first-token-detail",
            "tokensPerSec": 1176,
            "ttftMs": 6766
          },
          {
            "line": 2229,
            "rx": "2026-10-29T04:38:45.156Z",
            "sessionId": "f6de7310-09d4-4030-91db-2078960351ec",
            "source": "first-token-detail",
            "tokensPerSec": 1006,
            "ttftMs": 54830
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-find-vs-create",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-find-vs-create",
        "repetition": 1
      },
      "trialId": "craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-29T04-47-18-111Z-54nx",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-29T04-47-18-111Z-54nx/recording/chat-events.jsonl",
      "recordingSha256": "4d3dba29d1c95eb3c864ba33813113d79dcc07714704938fedfdd8801f5af57a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-29T04-47-18-111Z-54nx/metrics.json",
      "metricsSha256": "25dee6ea1e40fc1c7c0978bad34e0b7cec9fad0aeeec3cf821b7f47035780121",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 502,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-29T04:47:49.001Z",
            "sessionId": "8f2c2c46-3e82-4a62-b19a-8a90bb41a341",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 29277
          },
          {
            "line": 548,
            "rx": "2026-10-29T04:49:41.340Z",
            "sessionId": "ff1f2bd8-668d-407e-b865-9da9371c8ffa",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 26092
          },
          {
            "line": 3939,
            "rx": "2026-10-29T05:01:25.825Z",
            "sessionId": "8f2c2c46-3e82-4a62-b19a-8a90bb41a341",
            "source": "first-token-detail",
            "tokensPerSec": 303,
            "ttftMs": 2501
          },
          {
            "line": 4029,
            "rx": "2026-10-29T05:02:00.638Z",
            "sessionId": "ff1f2bd8-668d-407e-b865-9da9371c8ffa",
            "source": "first-token-detail",
            "tokensPerSec": 499,
            "ttftMs": 22758
          },
          {
            "line": 4346,
            "rx": "2026-10-29T05:03:02.427Z",
            "sessionId": "ff1f2bd8-668d-407e-b865-9da9371c8ffa",
            "source": "first-token-detail",
            "tokensPerSec": 458,
            "ttftMs": 1115
          },
          {
            "line": 4510,
            "rx": "2026-10-29T05:04:11.936Z",
            "sessionId": "ff1f2bd8-668d-407e-b865-9da9371c8ffa",
            "source": "first-token-detail",
            "tokensPerSec": 505,
            "ttftMs": 20266
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-edit-midtask",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-edit-midtask",
        "repetition": 1
      },
      "trialId": "craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-29T05-10-53-786Z-0wam",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-29T05-10-53-786Z-0wam/recording/chat-events.jsonl",
      "recordingSha256": "5175fb91a31cc81c434c30b3a6993e305f985990f2c8d0659f5d59ade5030796",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-29T05-10-53-786Z-0wam/metrics.json",
      "metricsSha256": "8bb1bbace21cce52e3567ce28a08796c826012d6182da7ea838ccdfe9b2d097f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 363.5,
        "sampleCount": 12,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 12,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 40,
            "rx": "2026-10-29T05:11:35.856Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 527,
            "ttftMs": 40373
          },
          {
            "line": 2583,
            "rx": "2026-10-29T05:19:05.703Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 260,
            "ttftMs": 1966
          },
          {
            "line": 5098,
            "rx": "2026-10-29T05:26:33.514Z",
            "sessionId": "26c4eb47-7348-4987-8deb-377214f38480",
            "source": "first-token-detail",
            "tokensPerSec": 452,
            "ttftMs": 26057
          },
          {
            "line": 5237,
            "rx": "2026-10-29T05:27:02.526Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 275,
            "ttftMs": 2245
          },
          {
            "line": 5547,
            "rx": "2026-10-29T05:28:00.290Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 240,
            "ttftMs": 1877
          },
          {
            "line": 5797,
            "rx": "2026-10-29T05:28:41.855Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 187,
            "ttftMs": 1305
          },
          {
            "line": 6330,
            "rx": "2026-10-29T05:30:16.068Z",
            "sessionId": "d336b0f0-893e-4436-bb54-df485fc9e760",
            "source": "first-token-detail",
            "tokensPerSec": 487,
            "ttftMs": 75386
          },
          {
            "line": 6355,
            "rx": "2026-10-29T05:30:27.270Z",
            "sessionId": "26c4eb47-7348-4987-8deb-377214f38480",
            "source": "first-token-detail",
            "tokensPerSec": 473,
            "ttftMs": 25208
          },
          {
            "line": 6738,
            "rx": "2026-10-29T05:31:42.536Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 208,
            "ttftMs": 1154
          },
          {
            "line": 6827,
            "rx": "2026-10-29T05:32:04.750Z",
            "sessionId": "26c4eb47-7348-4987-8deb-377214f38480",
            "source": "first-token-detail",
            "tokensPerSec": 474,
            "ttftMs": 9350
          },
          {
            "line": 6953,
            "rx": "2026-10-29T05:32:30.187Z",
            "sessionId": "d336b0f0-893e-4436-bb54-df485fc9e760",
            "source": "first-token-detail",
            "tokensPerSec": 460,
            "ttftMs": 14312
          },
          {
            "line": 6974,
            "rx": "2026-10-29T05:32:33.613Z",
            "sessionId": "1b0d681f-f6b4-4ff4-9139-a8d36ad50125",
            "source": "first-token-detail",
            "tokensPerSec": 179,
            "ttftMs": 11758
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-edit-midtask",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-edit-midtask",
        "repetition": 1
      },
      "trialId": "craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-29T05-57-06-675Z-sqvj",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-29T05-57-06-675Z-sqvj/recording/chat-events.jsonl",
      "recordingSha256": "3824def5c05720546479134a362d29a4c5b70e814dff1d875533843a18545650",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-29T05-57-06-675Z-sqvj/metrics.json",
      "metricsSha256": "048e33d4b3435ca9c6529963b624ff99987de16b842abce9f9cb117525d0dd7f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 736,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-29T05:57:28.823Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 959,
            "ttftMs": 20459
          },
          {
            "line": 236,
            "rx": "2026-10-29T05:58:11.884Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 566
          },
          {
            "line": 710,
            "rx": "2026-10-29T05:59:34.435Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 577,
            "ttftMs": 1003
          },
          {
            "line": 1209,
            "rx": "2026-10-29T06:01:13.211Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 475,
            "ttftMs": 623
          },
          {
            "line": 1551,
            "rx": "2026-10-29T06:02:09.546Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 885,
            "ttftMs": 494
          },
          {
            "line": 1703,
            "rx": "2026-10-29T06:02:32.775Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 386,
            "ttftMs": 536
          },
          {
            "line": 1986,
            "rx": "2026-10-29T06:03:26.280Z",
            "sessionId": "66ac8583-eec6-4cd2-9932-3a67e9d0e694",
            "source": "first-token-detail",
            "tokensPerSec": 956,
            "ttftMs": 4078
          },
          {
            "line": 2021,
            "rx": "2026-10-29T06:03:35.223Z",
            "sessionId": "f8237dfa-27eb-4561-a255-06a123db912e",
            "source": "first-token-detail",
            "tokensPerSec": 587,
            "ttftMs": 950
          },
          {
            "line": 2170,
            "rx": "2026-10-29T06:04:06.236Z",
            "sessionId": "053f9c90-f240-477f-a82f-40a5c4b99403",
            "source": "first-token-detail",
            "tokensPerSec": 1070,
            "ttftMs": 7981
          },
          {
            "line": 2300,
            "rx": "2026-10-29T06:04:40.633Z",
            "sessionId": "053f9c90-f240-477f-a82f-40a5c4b99403",
            "source": "first-token-detail",
            "tokensPerSec": 1141,
            "ttftMs": 7292
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-invoice-run",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-invoice-run",
        "repetition": 1
      },
      "trialId": "craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-29T06-05-23-199Z-bgk3",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-29T06-05-23-199Z-bgk3/recording/chat-events.jsonl",
      "recordingSha256": "45659a15df7aedd314e57b392b55f081ff16300c89275a4267bc184ad1d4e450",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-29T06-05-23-199Z-bgk3/metrics.json",
      "metricsSha256": "d411e9467ae242aea9131e2667bc7333f50ef241cb63915f46f54b835d52f829",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 561,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-29T06:05:50.432Z",
            "sessionId": "0f599b97-2125-4a14-b572-f3fb8a8f1007",
            "source": "first-token-detail",
            "tokensPerSec": 564,
            "ttftMs": 20727
          },
          {
            "line": 351,
            "rx": "2026-10-29T06:06:52.809Z",
            "sessionId": "34bb8723-2a47-4103-9ba1-bcb7dca104ca",
            "source": "first-token-detail",
            "tokensPerSec": 591,
            "ttftMs": 13039
          },
          {
            "line": 766,
            "rx": "2026-10-29T06:08:13.443Z",
            "sessionId": "3762fa77-1850-4bd1-94ef-9f542e6906b6",
            "source": "first-token-detail",
            "tokensPerSec": 566,
            "ttftMs": 13694
          },
          {
            "line": 1211,
            "rx": "2026-10-29T06:09:53.728Z",
            "sessionId": "60fdeb37-56cb-41bc-bc34-99614f2c7fd8",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 13918
          },
          {
            "line": 1695,
            "rx": "2026-10-29T06:11:41.235Z",
            "sessionId": "087f796d-eb66-44be-a31f-2c30bac5cce0",
            "source": "first-token-detail",
            "tokensPerSec": 511,
            "ttftMs": 16404
          },
          {
            "line": 2041,
            "rx": "2026-10-29T06:13:21.069Z",
            "sessionId": "d4d7c265-3d40-4629-aa7b-f40bad3827ff",
            "source": "first-token-detail",
            "tokensPerSec": 532,
            "ttftMs": 16192
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-invoice-run",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-invoice-run",
        "repetition": 1
      },
      "trialId": "craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-29T06-29-44-397Z-1hlf",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-29T06-29-44-397Z-1hlf/recording/chat-events.jsonl",
      "recordingSha256": "a33b8ab577f0e1b212dfb4cc1473b0d248b6607fbffb9213227c948365e2f060",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-29T06-29-44-397Z-1hlf/metrics.json",
      "metricsSha256": "4c9f2d3068ea8d6480f874f27083967436e6875fdfa60603efcc9249d1b5b2e0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 953,
        "sampleCount": 15,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 15,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-29T06:30:05.080Z",
            "sessionId": "f0c5717c-fc26-4221-b6cc-cbf4ce2648f2",
            "source": "first-token-detail",
            "tokensPerSec": 1080,
            "ttftMs": 14164
          },
          {
            "line": 129,
            "rx": "2026-10-29T06:30:32.877Z",
            "sessionId": "4ef93cdf-682a-44ba-b405-fdbbd6674416",
            "source": "first-token-detail",
            "tokensPerSec": 1208,
            "ttftMs": 6938
          },
          {
            "line": 368,
            "rx": "2026-10-29T06:31:36.931Z",
            "sessionId": "d0d77dd6-746a-4478-a786-2c189fc25d30",
            "source": "first-token-detail",
            "tokensPerSec": 1411,
            "ttftMs": 5976
          },
          {
            "line": 552,
            "rx": "2026-10-29T06:32:31.989Z",
            "sessionId": "939283f8-a001-48e2-842f-dcccc5437cb5",
            "source": "first-token-detail",
            "tokensPerSec": 1400,
            "ttftMs": 6022
          },
          {
            "line": 755,
            "rx": "2026-10-29T06:33:33.042Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 1251,
            "ttftMs": 7044
          },
          {
            "line": 2362,
            "rx": "2026-10-29T06:38:06.269Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 322,
            "ttftMs": 488
          },
          {
            "line": 2695,
            "rx": "2026-10-29T06:39:12.988Z",
            "sessionId": "bbf42640-2577-43f2-a2b0-69ef30c4d19c",
            "source": "first-token-detail",
            "tokensPerSec": 1244,
            "ttftMs": 6799
          },
          {
            "line": 4640,
            "rx": "2026-10-29T06:45:08.066Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 1011,
            "ttftMs": 18120
          },
          {
            "line": 4678,
            "rx": "2026-10-29T06:45:16.944Z",
            "sessionId": "8583600c-d363-4a73-be02-1c655275ce87",
            "source": "first-token-detail",
            "tokensPerSec": 284,
            "ttftMs": 30768
          },
          {
            "line": 4753,
            "rx": "2026-10-29T06:45:59.715Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 948,
            "ttftMs": 20004
          },
          {
            "line": 4847,
            "rx": "2026-10-29T06:46:31.558Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 953,
            "ttftMs": 21897
          },
          {
            "line": 5321,
            "rx": "2026-10-29T06:47:54.843Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 379,
            "ttftMs": 551
          },
          {
            "line": 5617,
            "rx": "2026-10-29T06:48:37.370Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 333,
            "ttftMs": 627
          },
          {
            "line": 5980,
            "rx": "2026-10-29T06:49:45.438Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 380,
            "ttftMs": 546
          },
          {
            "line": 6383,
            "rx": "2026-10-29T06:51:24.083Z",
            "sessionId": "acaa3d43-3b31-4922-a5fc-9b8de0580a7e",
            "source": "first-token-detail",
            "tokensPerSec": 863,
            "ttftMs": 42716
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-author-linear",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-linear",
        "repetition": 1
      },
      "trialId": "craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-29T07-33-23-937Z-njbf",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-29T07-33-23-937Z-njbf/recording/chat-events.jsonl",
      "recordingSha256": "6a0442b52cab77df146be4c3ddd4da1d69b8d1aca90a0ffa10b1281c9204f26b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-29T07-33-23-937Z-njbf/metrics.json",
      "metricsSha256": "c4abbf6782fbd380d5e8f11e070d5648dc150a2411b2899c6f04c9b00c55960b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 606.5,
        "sampleCount": 24,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 24,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-29T07:33:45.337Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 1031,
            "ttftMs": 19696
          },
          {
            "line": 351,
            "rx": "2026-10-29T07:34:56.108Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 351,
            "ttftMs": 432
          },
          {
            "line": 428,
            "rx": "2026-10-29T07:35:07.209Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 376,
            "ttftMs": 511
          },
          {
            "line": 614,
            "rx": "2026-10-29T07:35:42.671Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 327,
            "ttftMs": 579
          },
          {
            "line": 981,
            "rx": "2026-10-29T07:36:42.850Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 568,
            "ttftMs": 986
          },
          {
            "line": 1618,
            "rx": "2026-10-29T07:38:35.008Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 1012,
            "ttftMs": 5070
          },
          {
            "line": 1766,
            "rx": "2026-10-29T07:39:10.546Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 333,
            "ttftMs": 542
          },
          {
            "line": 1912,
            "rx": "2026-10-29T07:39:41.245Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 998,
            "ttftMs": 7599
          },
          {
            "line": 1959,
            "rx": "2026-10-29T07:39:52.722Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 439,
            "ttftMs": 701
          },
          {
            "line": 2152,
            "rx": "2026-10-29T07:40:28.268Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 1032,
            "ttftMs": 8067
          },
          {
            "line": 2437,
            "rx": "2026-10-29T07:41:14.025Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 455,
            "ttftMs": 507
          },
          {
            "line": 2687,
            "rx": "2026-10-29T07:41:53.880Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 323,
            "ttftMs": 552
          },
          {
            "line": 3116,
            "rx": "2026-10-29T07:43:02.728Z",
            "sessionId": "3ca95ad6-aade-4fc1-879e-3a39372bb35a",
            "source": "first-token-detail",
            "tokensPerSec": 499,
            "ttftMs": 585
          },
          {
            "line": 3401,
            "rx": "2026-10-29T07:43:48.657Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 338,
            "ttftMs": 492
          },
          {
            "line": 3587,
            "rx": "2026-10-29T07:44:26.324Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 1059,
            "ttftMs": 10614
          },
          {
            "line": 3648,
            "rx": "2026-10-29T07:47:38.938Z",
            "sessionId": "3c045e73-9745-4363-9a76-99cb19e6d4d1",
            "source": "first-token-detail",
            "tokensPerSec": 1778,
            "ttftMs": 7404
          },
          {
            "line": 3746,
            "rx": "2026-10-29T07:47:53.590Z",
            "sessionId": "3c045e73-9745-4363-9a76-99cb19e6d4d1",
            "source": "first-token-detail",
            "tokensPerSec": 951,
            "ttftMs": 1366
          },
          {
            "line": 3864,
            "rx": "2026-10-29T07:48:10.246Z",
            "sessionId": "28c1eab5-b68f-44be-a8b7-adb25120c6f3",
            "source": "first-token-detail",
            "tokensPerSec": 500,
            "ttftMs": 19358
          },
          {
            "line": 4000,
            "rx": "2026-10-29T07:48:32.348Z",
            "sessionId": "3c045e73-9745-4363-9a76-99cb19e6d4d1",
            "source": "first-token-detail",
            "tokensPerSec": 682,
            "ttftMs": 548
          },
          {
            "line": 4231,
            "rx": "2026-10-29T07:49:05.502Z",
            "sessionId": "3c045e73-9745-4363-9a76-99cb19e6d4d1",
            "source": "first-token-detail",
            "tokensPerSec": 678,
            "ttftMs": 725
          },
          {
            "line": 4374,
            "rx": "2026-10-29T07:49:24.862Z",
            "sessionId": "3c045e73-9745-4363-9a76-99cb19e6d4d1",
            "source": "first-token-detail",
            "tokensPerSec": 645,
            "ttftMs": 674
          },
          {
            "line": 4594,
            "rx": "2026-10-29T07:49:55.685Z",
            "sessionId": "28c1eab5-b68f-44be-a8b7-adb25120c6f3",
            "source": "first-token-detail",
            "tokensPerSec": 927,
            "ttftMs": 1039
          },
          {
            "line": 4751,
            "rx": "2026-10-29T07:50:22.841Z",
            "sessionId": "42003214-5c1f-47d5-bac9-fba6e64e0651",
            "source": "first-token-detail",
            "tokensPerSec": 321,
            "ttftMs": 28886
          },
          {
            "line": 4805,
            "rx": "2026-10-29T07:50:32.618Z",
            "sessionId": "3c045e73-9745-4363-9a76-99cb19e6d4d1",
            "source": "first-token-detail",
            "tokensPerSec": 819,
            "ttftMs": 478
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-author-linear",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-linear",
        "repetition": 1
      },
      "trialId": "craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-29T08-00-04-504Z-oagz",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-29T08-00-04-504Z-oagz/recording/chat-events.jsonl",
      "recordingSha256": "7147dc9d87e61164974bd7b0b4f3f53b1acc381c1184114a1cc1041c8a9c027b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-29T08-00-04-504Z-oagz/metrics.json",
      "metricsSha256": "2ed2a944153318f44caf774376e93eb5d30c0a35d5508f41895c7eb3d35dc7f6",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 495,
        "sampleCount": 11,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 11,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 37,
            "rx": "2026-10-29T08:00:46.008Z",
            "sessionId": "d87b944e-9e18-4219-a3d1-602b70467dbd",
            "source": "first-token-detail",
            "tokensPerSec": 534,
            "ttftMs": 39831
          },
          {
            "line": 5212,
            "rx": "2026-10-29T08:15:57.453Z",
            "sessionId": "d87b944e-9e18-4219-a3d1-602b70467dbd",
            "source": "first-token-detail",
            "tokensPerSec": 247,
            "ttftMs": 2085
          },
          {
            "line": 6558,
            "rx": "2026-10-29T08:19:46.389Z",
            "sessionId": "a6ae8441-9af4-4227-8271-f6074b251302",
            "source": "first-token-detail",
            "tokensPerSec": 495,
            "ttftMs": 209754
          },
          {
            "line": 7887,
            "rx": "2026-10-29T08:23:34.463Z",
            "sessionId": "d87b944e-9e18-4219-a3d1-602b70467dbd",
            "source": "first-token-detail",
            "tokensPerSec": 252,
            "ttftMs": 35868
          },
          {
            "line": 8341,
            "rx": "2026-10-29T08:25:00.097Z",
            "sessionId": "68024724-92eb-4b1c-a714-54a87b3431bf",
            "source": "first-token-detail",
            "tokensPerSec": 498,
            "ttftMs": 8799
          },
          {
            "line": 8593,
            "rx": "2026-10-29T08:25:56.569Z",
            "sessionId": "d2ad12cc-bf7c-4cee-99bc-b2d6f9e01e85",
            "source": "first-token-detail",
            "tokensPerSec": 506,
            "ttftMs": 29930
          },
          {
            "line": 9104,
            "rx": "2026-10-29T08:27:20.113Z",
            "sessionId": "d87b944e-9e18-4219-a3d1-602b70467dbd",
            "source": "first-token-detail",
            "tokensPerSec": 258,
            "ttftMs": 32913
          },
          {
            "line": 10149,
            "rx": "2026-10-29T08:30:23.471Z",
            "sessionId": "68024724-92eb-4b1c-a714-54a87b3431bf",
            "source": "first-token-detail",
            "tokensPerSec": 276,
            "ttftMs": 1022
          },
          {
            "line": 10584,
            "rx": "2026-10-29T08:31:45.104Z",
            "sessionId": "62307e83-8b70-40d8-a641-3f2b9d2b7d88",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 68228
          },
          {
            "line": 11276,
            "rx": "2026-10-29T08:33:41.626Z",
            "sessionId": "d87b944e-9e18-4219-a3d1-602b70467dbd",
            "source": "first-token-detail",
            "tokensPerSec": 213,
            "ttftMs": 6812
          },
          {
            "line": 12029,
            "rx": "2026-10-29T08:36:07.037Z",
            "sessionId": "68024724-92eb-4b1c-a714-54a87b3431bf",
            "source": "first-token-detail",
            "tokensPerSec": 509,
            "ttftMs": 31931
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-route-multi",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-route-multi",
        "repetition": 1
      },
      "trialId": "craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-29T08-36-51-158Z-69bz",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-29T08-36-51-158Z-69bz/recording/chat-events.jsonl",
      "recordingSha256": "2fb9a5ba4228f3d3fb6a6c062d0f08cbc2a1d0cf1a6f34b7aa6c6c722d5601d2",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-29T08-36-51-158Z-69bz/metrics.json",
      "metricsSha256": "41c4ab4567b071c606392196317a55f17bca6fd565c3c6c70ad14a36faf4e8bb",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 880,
        "sampleCount": 15,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 15,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-29T08:37:08.355Z",
            "sessionId": "6b06a1f7-86af-4b44-9c7e-5d29c445bdc8",
            "source": "first-token-detail",
            "tokensPerSec": 1105,
            "ttftMs": 15470
          },
          {
            "line": 1526,
            "rx": "2026-10-29T08:41:27.697Z",
            "sessionId": "6b06a1f7-86af-4b44-9c7e-5d29c445bdc8",
            "source": "first-token-detail",
            "tokensPerSec": 224,
            "ttftMs": 467
          },
          {
            "line": 3163,
            "rx": "2026-10-29T08:46:02.290Z",
            "sessionId": "6b06a1f7-86af-4b44-9c7e-5d29c445bdc8",
            "source": "first-token-detail",
            "tokensPerSec": 261,
            "ttftMs": 424
          },
          {
            "line": 4838,
            "rx": "2026-10-29T08:50:42.788Z",
            "sessionId": "6b06a1f7-86af-4b44-9c7e-5d29c445bdc8",
            "source": "first-token-detail",
            "tokensPerSec": 351,
            "ttftMs": 452
          },
          {
            "line": 5160,
            "rx": "2026-10-29T08:51:32.713Z",
            "sessionId": "6b06a1f7-86af-4b44-9c7e-5d29c445bdc8",
            "source": "first-token-detail",
            "tokensPerSec": 447,
            "ttftMs": 560
          },
          {
            "line": 5556,
            "rx": "2026-10-29T08:52:40.116Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 1025,
            "ttftMs": 3852
          },
          {
            "line": 5627,
            "rx": "2026-10-29T08:53:00.169Z",
            "sessionId": "6b06a1f7-86af-4b44-9c7e-5d29c445bdc8",
            "source": "first-token-detail",
            "tokensPerSec": 556,
            "ttftMs": 882
          },
          {
            "line": 5896,
            "rx": "2026-10-29T08:53:44.206Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 1028,
            "ttftMs": 5645
          },
          {
            "line": 6003,
            "rx": "2026-10-29T08:54:30.249Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 1266,
            "ttftMs": 5413
          },
          {
            "line": 6061,
            "rx": "2026-10-29T08:54:51.090Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 965,
            "ttftMs": 9446
          },
          {
            "line": 6150,
            "rx": "2026-10-29T08:55:04.437Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 434,
            "ttftMs": 512
          },
          {
            "line": 6333,
            "rx": "2026-10-29T08:55:37.188Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 452,
            "ttftMs": 414
          },
          {
            "line": 6514,
            "rx": "2026-10-29T08:56:13.955Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 1021,
            "ttftMs": 9539
          },
          {
            "line": 6554,
            "rx": "2026-10-29T08:56:37.352Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 1154,
            "ttftMs": 11150
          },
          {
            "line": 6589,
            "rx": "2026-10-29T08:56:46.802Z",
            "sessionId": "a261a85c-c93a-4eb5-a036-e80fb4ca2fb4",
            "source": "first-token-detail",
            "tokensPerSec": 880,
            "ttftMs": 1013
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-route-multi",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-route-multi",
        "repetition": 1
      },
      "trialId": "craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-29T09-04-39-277Z-5er1",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-29T09-04-39-277Z-5er1/recording/chat-events.jsonl",
      "recordingSha256": "72d4de710f4ecf45b96de974c0f12438b43d413e4fbf27e9383df1607fe1347b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-29T09-04-39-277Z-5er1/metrics.json",
      "metricsSha256": "1a89ce67a62e6f26b69482b6b570784f52e9e9e834fb401f0673b231184606b6",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 472,
        "sampleCount": 19,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 19,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-29T09:05:09.813Z",
            "sessionId": "e4ad0886-cf7b-4008-a605-27261a03edae",
            "source": "first-token-detail",
            "tokensPerSec": 534,
            "ttftMs": 28815
          },
          {
            "line": 809,
            "rx": "2026-10-29T09:07:27.994Z",
            "sessionId": "e4ad0886-cf7b-4008-a605-27261a03edae",
            "source": "first-token-detail",
            "tokensPerSec": 284,
            "ttftMs": 1169
          },
          {
            "line": 1037,
            "rx": "2026-10-29T09:08:23.697Z",
            "sessionId": "e4ad0886-cf7b-4008-a605-27261a03edae",
            "source": "first-token-detail",
            "tokensPerSec": 267,
            "ttftMs": 1490
          },
          {
            "line": 1082,
            "rx": "2026-10-29T09:08:30.504Z",
            "sessionId": "4a2ecb72-6b0b-4a4a-a23a-e25a2198673b",
            "source": "first-token-detail",
            "tokensPerSec": 140,
            "ttftMs": 59538
          },
          {
            "line": 1320,
            "rx": "2026-10-29T09:09:55.432Z",
            "sessionId": "7925eb7f-1e13-4271-a65b-e15ea483b2e4",
            "source": "first-token-detail",
            "tokensPerSec": 506,
            "ttftMs": 35884
          },
          {
            "line": 1809,
            "rx": "2026-10-29T09:11:40.354Z",
            "sessionId": "6e2c24f7-ebbf-442f-8099-bd1f1a2875c6",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 23484
          },
          {
            "line": 2023,
            "rx": "2026-10-29T09:12:26.158Z",
            "sessionId": "6e2c24f7-ebbf-442f-8099-bd1f1a2875c6",
            "source": "first-token-detail",
            "tokensPerSec": 359,
            "ttftMs": 5391
          },
          {
            "line": 2164,
            "rx": "2026-10-29T09:12:47.668Z",
            "sessionId": "00946c3b-dabd-4612-974e-b768ad4a015b",
            "source": "first-token-detail",
            "tokensPerSec": 145,
            "ttftMs": 56729
          },
          {
            "line": 3704,
            "rx": "2026-10-29T09:17:29.185Z",
            "sessionId": "b0fe00f6-51d0-4fbd-91c0-a071030da3a5",
            "source": "first-token-detail",
            "tokensPerSec": 499,
            "ttftMs": 37997
          },
          {
            "line": 4886,
            "rx": "2026-10-29T09:20:49.320Z",
            "sessionId": "6e2c24f7-ebbf-442f-8099-bd1f1a2875c6",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 15851
          },
          {
            "line": 5214,
            "rx": "2026-10-29T09:21:48.418Z",
            "sessionId": "e4ad0886-cf7b-4008-a605-27261a03edae",
            "source": "first-token-detail",
            "tokensPerSec": 262,
            "ttftMs": 1914
          },
          {
            "line": 5733,
            "rx": "2026-10-29T09:23:24.305Z",
            "sessionId": "b0fe00f6-51d0-4fbd-91c0-a071030da3a5",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 37905
          },
          {
            "line": 5799,
            "rx": "2026-10-29T09:23:34.853Z",
            "sessionId": "7925eb7f-1e13-4271-a65b-e15ea483b2e4",
            "source": "first-token-detail",
            "tokensPerSec": 330,
            "ttftMs": 33016
          },
          {
            "line": 6540,
            "rx": "2026-10-29T09:25:38.217Z",
            "sessionId": "6e2c24f7-ebbf-442f-8099-bd1f1a2875c6",
            "source": "first-token-detail",
            "tokensPerSec": 479,
            "ttftMs": 41980
          },
          {
            "line": 6729,
            "rx": "2026-10-29T09:26:17.146Z",
            "sessionId": "e4ad0886-cf7b-4008-a605-27261a03edae",
            "source": "first-token-detail",
            "tokensPerSec": 249,
            "ttftMs": 1394
          },
          {
            "line": 6823,
            "rx": "2026-10-29T09:26:33.615Z",
            "sessionId": "b9ef361e-3607-427a-a75e-154c3bd358cf",
            "source": "first-token-detail",
            "tokensPerSec": 472,
            "ttftMs": 5897
          },
          {
            "line": 6917,
            "rx": "2026-10-29T09:26:58.342Z",
            "sessionId": "20c79d74-4093-41cf-98b1-0dcfadb0c022",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 27044
          },
          {
            "line": 6971,
            "rx": "2026-10-29T09:27:14.635Z",
            "sessionId": "6e2c24f7-ebbf-442f-8099-bd1f1a2875c6",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 31322
          },
          {
            "line": 7354,
            "rx": "2026-10-29T09:28:21.992Z",
            "sessionId": "e4ad0886-cf7b-4008-a605-27261a03edae",
            "source": "first-token-detail",
            "tokensPerSec": 221,
            "ttftMs": 4508
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-export-generalize",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-export-generalize",
        "repetition": 1
      },
      "trialId": "craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-29T10-20-54-840Z-oj3a",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-29T10-20-54-840Z-oj3a/recording/chat-events.jsonl",
      "recordingSha256": "6c6ad9ad0cacd3f5f66799097ac5e32965280da001a68102775e62c5153d4515",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-29T10-20-54-840Z-oj3a/metrics.json",
      "metricsSha256": "cba4d6c01e8824868b4b08a8b9a1791725d106a6bb16712cb4c6b18f0bf599ee",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 532,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 35,
            "rx": "2026-10-29T10:21:26.492Z",
            "sessionId": "2ff35645-ad92-4772-9e4b-2cd56bb3260c",
            "source": "first-token-detail",
            "tokensPerSec": 760,
            "ttftMs": 30111
          },
          {
            "line": 3334,
            "rx": "2026-10-29T10:31:36.303Z",
            "sessionId": "2ff35645-ad92-4772-9e4b-2cd56bb3260c",
            "source": "first-token-detail",
            "tokensPerSec": 378,
            "ttftMs": 820
          },
          {
            "line": 4909,
            "rx": "2026-10-29T10:36:07.548Z",
            "sessionId": "aeafd443-2f1a-4117-89cf-ccab2810beb9",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 9951
          },
          {
            "line": 5145,
            "rx": "2026-10-29T10:36:45.706Z",
            "sessionId": "2ff35645-ad92-4772-9e4b-2cd56bb3260c",
            "source": "first-token-detail",
            "tokensPerSec": 231,
            "ttftMs": 1595
          },
          {
            "line": 6135,
            "rx": "2026-10-29T10:39:32.325Z",
            "sessionId": "aeafd443-2f1a-4117-89cf-ccab2810beb9",
            "source": "first-token-detail",
            "tokensPerSec": 328,
            "ttftMs": 1596
          },
          {
            "line": 6480,
            "rx": "2026-10-29T10:40:30.876Z",
            "sessionId": "2ff35645-ad92-4772-9e4b-2cd56bb3260c",
            "source": "first-token-detail",
            "tokensPerSec": 283,
            "ttftMs": 2267
          },
          {
            "line": 6752,
            "rx": "2026-10-29T10:49:30.380Z",
            "sessionId": "2ff35645-ad92-4772-9e4b-2cd56bb3260c",
            "source": "first-token-detail",
            "tokensPerSec": 563,
            "ttftMs": 7825
          },
          {
            "line": 13008,
            "rx": "2026-10-29T11:12:39.689Z",
            "sessionId": "475ee227-5ca9-45eb-a6b3-1fd244c0b4ae",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 381906
          },
          {
            "line": 18432,
            "rx": "2026-10-29T11:29:57.053Z",
            "sessionId": "5ffec607-80a8-4600-8a5d-e79e8d7fe4dc",
            "source": "first-token-detail",
            "tokensPerSec": 569,
            "ttftMs": 14109
          },
          {
            "line": 19313,
            "rx": "2026-10-29T11:32:47.023Z",
            "sessionId": "2ff35645-ad92-4772-9e4b-2cd56bb3260c",
            "source": "first-token-detail",
            "tokensPerSec": 396,
            "ttftMs": 9719
          },
          {
            "line": 20573,
            "rx": "2026-10-29T11:37:25.918Z",
            "sessionId": "ad1c8aaa-9b7f-40f1-b1e1-75c73a9c18fb",
            "source": "first-token-detail",
            "tokensPerSec": 521,
            "ttftMs": 227299
          },
          {
            "line": 26381,
            "rx": "2026-10-29T11:59:38.164Z",
            "sessionId": "8d641681-e9c8-4db2-a151-5d33a6b19304",
            "source": "first-token-detail",
            "tokensPerSec": 552,
            "ttftMs": 14572
          },
          {
            "line": 28550,
            "rx": "2026-10-29T12:06:10.414Z",
            "sessionId": "25d94cac-ec93-4180-a314-a29e2848eef5",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 11806
          },
          {
            "line": 28869,
            "rx": "2026-10-29T12:07:14.031Z",
            "sessionId": "91c04923-e58e-44b5-90f7-55ed02907378",
            "source": "first-token-detail",
            "tokensPerSec": 532,
            "ttftMs": 15291
          },
          {
            "line": 30929,
            "rx": "2026-10-29T12:14:13.047Z",
            "sessionId": "1668ed90-6c5b-47a1-a6ee-8a8712dffe91",
            "source": "first-token-detail",
            "tokensPerSec": 882,
            "ttftMs": 4125
          },
          {
            "line": 31337,
            "rx": "2026-10-29T12:16:22.355Z",
            "sessionId": "ea2d9207-278e-44ce-ae79-fa8e6743a0a9",
            "source": "first-token-detail",
            "tokensPerSec": 750,
            "ttftMs": 3401
          },
          {
            "line": 31456,
            "rx": "2026-10-29T12:19:52.991Z",
            "sessionId": "e02f4477-b7df-4e80-91a1-7082c87e645a",
            "source": "first-token-detail",
            "tokensPerSec": 779,
            "ttftMs": 21393
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-export-generalize",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-export-generalize",
        "repetition": 1
      },
      "trialId": "craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-29T12-56-00-732Z-h88c",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-29T12-56-00-732Z-h88c/recording/chat-events.jsonl",
      "recordingSha256": "0c3b63c94031c32546aa84979c1210d5cd2d0ac9cf017cc8d2c97a4986e7f145",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-29T12-56-00-732Z-h88c/metrics.json",
      "metricsSha256": "120d09e09f2bc68ee6709b3edc63086d6597a576e0469c2f510bdf8443d2ec08",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 490,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-29T12:56:25.154Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 856,
            "ttftMs": 22762
          },
          {
            "line": 432,
            "rx": "2026-10-29T12:57:59.555Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 417,
            "ttftMs": 741
          },
          {
            "line": 662,
            "rx": "2026-10-29T13:06:48.924Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 884,
            "ttftMs": 6570
          },
          {
            "line": 904,
            "rx": "2026-10-29T13:07:26.026Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 500,
            "ttftMs": 305
          },
          {
            "line": 1186,
            "rx": "2026-10-29T13:08:19.952Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 644,
            "ttftMs": 492
          },
          {
            "line": 1267,
            "rx": "2026-10-29T13:08:33.659Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 988,
            "ttftMs": 549
          },
          {
            "line": 1513,
            "rx": "2026-10-29T13:09:18.432Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 490,
            "ttftMs": 513
          },
          {
            "line": 1663,
            "rx": "2026-10-29T13:09:41.231Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 525,
            "ttftMs": 534
          },
          {
            "line": 1777,
            "rx": "2026-10-29T13:10:00.472Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 406,
            "ttftMs": 616
          },
          {
            "line": 1961,
            "rx": "2026-10-29T13:10:28.557Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 428,
            "ttftMs": 599
          },
          {
            "line": 2194,
            "rx": "2026-10-29T13:11:20.567Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 982,
            "ttftMs": 1093
          },
          {
            "line": 3049,
            "rx": "2026-10-29T13:13:46.226Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 324,
            "ttftMs": 645
          },
          {
            "line": 3251,
            "rx": "2026-10-29T13:14:18.260Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 629,
            "ttftMs": 1491
          },
          {
            "line": 3486,
            "rx": "2026-10-29T13:15:04.671Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 306,
            "ttftMs": 464
          },
          {
            "line": 3737,
            "rx": "2026-10-29T13:15:45.146Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 323,
            "ttftMs": 990
          },
          {
            "line": 4399,
            "rx": "2026-10-29T13:17:34.630Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 428,
            "ttftMs": 793
          },
          {
            "line": 4523,
            "rx": "2026-10-29T13:17:53.155Z",
            "sessionId": "5fe65d9c-1a9a-437c-a3ec-5d03723594a9",
            "source": "first-token-detail",
            "tokensPerSec": 432,
            "ttftMs": 790
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-author-params",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-params",
        "repetition": 1
      },
      "trialId": "craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-29T13-19-36-196Z-24u6",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-29T13-19-36-196Z-24u6/recording/chat-events.jsonl",
      "recordingSha256": "4831f70e6c9076718167e6fc116f71808a86437d1bd6b0cc50f9df8cd12f9aaa",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-29T13-19-36-196Z-24u6/metrics.json",
      "metricsSha256": "890a5b138f4c826a322c863341e68e45cecd739099be9af1dbe968879a3a34ab",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 509,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 38,
            "rx": "2026-10-29T13:20:19.299Z",
            "sessionId": "dee2bf35-d079-462e-808e-15a016ceedc4",
            "source": "first-token-detail",
            "tokensPerSec": 521,
            "ttftMs": 41313
          },
          {
            "line": 2481,
            "rx": "2026-10-29T13:28:02.530Z",
            "sessionId": "f3d8d4dd-9a70-4327-bc5e-623e83ec4ae6",
            "source": "first-token-detail",
            "tokensPerSec": 231,
            "ttftMs": 69311
          },
          {
            "line": 2586,
            "rx": "2026-10-29T13:28:19.763Z",
            "sessionId": "dee2bf35-d079-462e-808e-15a016ceedc4",
            "source": "first-token-detail",
            "tokensPerSec": 295,
            "ttftMs": 26239
          },
          {
            "line": 6532,
            "rx": "2026-10-29T13:40:28.553Z",
            "sessionId": "654624c0-c79a-4c9f-8777-e2931e6c16c7",
            "source": "first-token-detail",
            "tokensPerSec": 509,
            "ttftMs": 210184
          },
          {
            "line": 7604,
            "rx": "2026-10-29T13:43:38.999Z",
            "sessionId": "dee2bf35-d079-462e-808e-15a016ceedc4",
            "source": "first-token-detail",
            "tokensPerSec": 250,
            "ttftMs": 18571
          },
          {
            "line": 8095,
            "rx": "2026-10-29T13:45:27.745Z",
            "sessionId": "01fec0ca-1b20-4df0-ae12-7d8935dc2a3d",
            "source": "first-token-detail",
            "tokensPerSec": 505,
            "ttftMs": 109410
          },
          {
            "line": 9690,
            "rx": "2026-10-29T13:50:51.046Z",
            "sessionId": "d128e57c-a6a2-4763-94b4-e9d0fe0ae377",
            "source": "first-token-detail",
            "tokensPerSec": 559,
            "ttftMs": 23505
          },
          {
            "line": 9863,
            "rx": "2026-10-29T13:51:20.946Z",
            "sessionId": "6540d2a8-136e-4ca5-b73f-2ecd149ba0bb",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 7274
          },
          {
            "line": 9883,
            "rx": "2026-10-29T13:51:29.933Z",
            "sessionId": "b00f785a-b7aa-40a5-83f3-9d90265bd42c",
            "source": "first-token-detail",
            "tokensPerSec": 535,
            "ttftMs": 15832
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-author-params",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-params",
        "repetition": 1
      },
      "trialId": "craftbook-author-params-mlx-gemma4-12b-q4-2026-10-29T14-12-35-123Z-ovs1",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-gemma4-12b-q4-2026-10-29T14-12-35-123Z-ovs1/recording/chat-events.jsonl",
      "recordingSha256": "5c3d05adf741b0350befb09ad1982a658c42f5946028b01b494db79c1ce266a0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-gemma4-12b-q4-2026-10-29T14-12-35-123Z-ovs1/metrics.json",
      "metricsSha256": "db2cfe652dbee8dcda7210ab5943f8fe2f329d56643c56b092e562567413978b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 461.5,
        "sampleCount": 18,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 18,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-29T14:12:56.422Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 1040,
            "ttftMs": 19622
          },
          {
            "line": 289,
            "rx": "2026-10-29T14:13:56.680Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 482
          },
          {
            "line": 436,
            "rx": "2026-10-29T14:14:18.490Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 480,
            "ttftMs": 516
          },
          {
            "line": 517,
            "rx": "2026-10-29T14:14:30.273Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 380,
            "ttftMs": 431
          },
          {
            "line": 666,
            "rx": "2026-10-29T14:14:53.822Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 419,
            "ttftMs": 526
          },
          {
            "line": 864,
            "rx": "2026-10-29T14:15:27.224Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 481,
            "ttftMs": 1019
          },
          {
            "line": 1193,
            "rx": "2026-10-29T14:16:24.060Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 390,
            "ttftMs": 545
          },
          {
            "line": 1317,
            "rx": "2026-10-29T14:16:42.687Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 387,
            "ttftMs": 527
          },
          {
            "line": 1483,
            "rx": "2026-10-29T14:17:08.271Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 350,
            "ttftMs": 505
          },
          {
            "line": 1660,
            "rx": "2026-10-29T14:17:39.051Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 993,
            "ttftMs": 532
          },
          {
            "line": 2061,
            "rx": "2026-10-29T14:18:46.756Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 458,
            "ttftMs": 526
          },
          {
            "line": 2163,
            "rx": "2026-10-29T14:19:01.728Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 433,
            "ttftMs": 538
          },
          {
            "line": 2350,
            "rx": "2026-10-29T14:21:00.769Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 720,
            "ttftMs": 548
          },
          {
            "line": 2743,
            "rx": "2026-10-29T14:22:01.331Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 1165,
            "ttftMs": 677
          },
          {
            "line": 2970,
            "rx": "2026-10-29T14:22:39.065Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 568,
            "ttftMs": 408
          },
          {
            "line": 3134,
            "rx": "2026-10-29T14:23:04.061Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 465,
            "ttftMs": 571
          },
          {
            "line": 3786,
            "rx": "2026-10-29T14:24:48.052Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 831,
            "ttftMs": 1074
          },
          {
            "line": 4184,
            "rx": "2026-10-29T14:26:00.386Z",
            "sessionId": "7f2caa31-8ff0-4c04-9e0c-a1619e15352a",
            "source": "first-token-detail",
            "tokensPerSec": 383,
            "ttftMs": 739
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-author-gate-script",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-gate-script",
        "repetition": 1
      },
      "trialId": "craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-29T18-15-52-291Z-hb1t",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-29T18-15-52-291Z-hb1t/recording/chat-events.jsonl",
      "recordingSha256": "1e2aa00671f780db5f09a49d00541e50a83769880a822bee0019db53abf8402e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-29T18-15-52-291Z-hb1t/metrics.json",
      "metricsSha256": "6d740f372273bcfcbced95a307cadc9f2907b52e4b0ab04f4ca4f9561596b16a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 585.5,
        "sampleCount": 16,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 16,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-29T18:16:13.682Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 1047,
            "ttftMs": 19479
          },
          {
            "line": 305,
            "rx": "2026-10-29T18:17:24.032Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 389,
            "ttftMs": 402
          },
          {
            "line": 373,
            "rx": "2026-10-29T18:17:33.471Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 574,
            "ttftMs": 731
          },
          {
            "line": 702,
            "rx": "2026-10-29T18:18:33.240Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 496,
            "ttftMs": 635
          },
          {
            "line": 859,
            "rx": "2026-10-29T18:19:00.733Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 445,
            "ttftMs": 806
          },
          {
            "line": 1043,
            "rx": "2026-10-29T18:19:33.904Z",
            "sessionId": "c4c43166-d550-4452-9e67-659f07c6af28",
            "source": "first-token-detail",
            "tokensPerSec": 1038,
            "ttftMs": 3654
          },
          {
            "line": 1087,
            "rx": "2026-10-29T18:19:45.734Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 597,
            "ttftMs": 608
          },
          {
            "line": 1214,
            "rx": "2026-10-29T18:20:08.074Z",
            "sessionId": "c4c43166-d550-4452-9e67-659f07c6af28",
            "source": "first-token-detail",
            "tokensPerSec": 1003,
            "ttftMs": 5831
          },
          {
            "line": 1232,
            "rx": "2026-10-29T18:20:11.817Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 386,
            "ttftMs": 476
          },
          {
            "line": 1400,
            "rx": "2026-10-29T18:20:42.547Z",
            "sessionId": "c4c43166-d550-4452-9e67-659f07c6af28",
            "source": "first-token-detail",
            "tokensPerSec": 1005,
            "ttftMs": 6722
          },
          {
            "line": 1473,
            "rx": "2026-10-29T18:21:01.138Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 455,
            "ttftMs": 596
          },
          {
            "line": 1498,
            "rx": "2026-10-29T18:21:08.300Z",
            "sessionId": "c4c43166-d550-4452-9e67-659f07c6af28",
            "source": "first-token-detail",
            "tokensPerSec": 990,
            "ttftMs": 6409
          },
          {
            "line": 1704,
            "rx": "2026-10-29T18:21:48.721Z",
            "sessionId": "c4c43166-d550-4452-9e67-659f07c6af28",
            "source": "first-token-detail",
            "tokensPerSec": 992,
            "ttftMs": 8224
          },
          {
            "line": 1728,
            "rx": "2026-10-29T18:21:52.882Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 535,
            "ttftMs": 645
          },
          {
            "line": 1797,
            "rx": "2026-10-29T18:22:08.438Z",
            "sessionId": "c4c43166-d550-4452-9e67-659f07c6af28",
            "source": "first-token-detail",
            "tokensPerSec": 972,
            "ttftMs": 7748
          },
          {
            "line": 1849,
            "rx": "2026-10-29T18:22:22.362Z",
            "sessionId": "304c38b2-2ee1-4b9e-8226-d25d44c85814",
            "source": "first-token-detail",
            "tokensPerSec": 479,
            "ttftMs": 450
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-author-gate-script",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-gate-script",
        "repetition": 1
      },
      "trialId": "craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-29T18-45-30-573Z-tp7x",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-29T18-45-30-573Z-tp7x/recording/chat-events.jsonl",
      "recordingSha256": "807edb81406461996a91a45e555db0a711c876dd6866c1923c9158709cfa1947",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-29T18-45-30-573Z-tp7x/metrics.json",
      "metricsSha256": "dce32654c161d49f257362cca62865ef0c600fb13b062ed89bd1ab0336d15fe3",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 330,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 37,
            "rx": "2026-10-29T18:46:10.067Z",
            "sessionId": "f72362ce-1c9c-4a6f-8c24-4245b5722e2b",
            "source": "first-token-detail",
            "tokensPerSec": 584,
            "ttftMs": 37810
          },
          {
            "line": 14747,
            "rx": "2026-10-29T19:39:42.581Z",
            "sessionId": "f72362ce-1c9c-4a6f-8c24-4245b5722e2b",
            "source": "first-token-detail",
            "tokensPerSec": 76,
            "ttftMs": 1522
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-gemma-mlx-complex-work-craftbook-author-fanout",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-fanout",
        "repetition": 1
      },
      "trialId": "craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-29T19-43-48-872Z-euv1",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-29T19-43-48-872Z-euv1/recording/chat-events.jsonl",
      "recordingSha256": "8470ff908b5d3fcb7cd53c01b42a92086af7837f3be5ebd3bac2ea3fc7d68a86",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-gemma-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-29T19-43-48-872Z-euv1/metrics.json",
      "metricsSha256": "d2fe65a46598164812527f001f4ac0799350656b7bd2f05221e562926b115e2f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 652,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-29T19:44:09.700Z",
            "sessionId": "3c4f9470-7ff4-4389-ba0f-988b652b41ab",
            "source": "first-token-detail",
            "tokensPerSec": 1078,
            "ttftMs": 19157
          },
          {
            "line": 565,
            "rx": "2026-10-29T19:46:00.992Z",
            "sessionId": "3c4f9470-7ff4-4389-ba0f-988b652b41ab",
            "source": "first-token-detail",
            "tokensPerSec": 634,
            "ttftMs": 953
          },
          {
            "line": 794,
            "rx": "2026-10-29T19:46:41.785Z",
            "sessionId": "3c4f9470-7ff4-4389-ba0f-988b652b41ab",
            "source": "first-token-detail",
            "tokensPerSec": 1049,
            "ttftMs": 511
          },
          {
            "line": 1164,
            "rx": "2026-10-29T19:47:50.767Z",
            "sessionId": "3c4f9470-7ff4-4389-ba0f-988b652b41ab",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 486
          },
          {
            "line": 1309,
            "rx": "2026-10-29T19:48:12.826Z",
            "sessionId": "3c4f9470-7ff4-4389-ba0f-988b652b41ab",
            "source": "first-token-detail",
            "tokensPerSec": 364,
            "ttftMs": 519
          },
          {
            "line": 1429,
            "rx": "2026-10-29T19:54:33.658Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 1783,
            "ttftMs": 13148
          },
          {
            "line": 1485,
            "rx": "2026-10-29T19:54:41.594Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 1006,
            "ttftMs": 764
          },
          {
            "line": 1593,
            "rx": "2026-10-29T19:54:58.219Z",
            "sessionId": "5ea0d65b-2cbb-4d65-b2f5-93baeacaf764",
            "source": "first-token-detail",
            "tokensPerSec": 513,
            "ttftMs": 19283
          },
          {
            "line": 1735,
            "rx": "2026-10-29T19:55:19.707Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 659,
            "ttftMs": 405
          },
          {
            "line": 2030,
            "rx": "2026-10-29T19:56:04.330Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 676,
            "ttftMs": 728
          },
          {
            "line": 2205,
            "rx": "2026-10-29T19:56:29.488Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 491
          },
          {
            "line": 2327,
            "rx": "2026-10-29T19:56:45.682Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 652,
            "ttftMs": 636
          },
          {
            "line": 2593,
            "rx": "2026-10-29T19:57:22.855Z",
            "sessionId": "632cef98-e520-4432-988d-fc940ccfd1e1",
            "source": "first-token-detail",
            "tokensPerSec": 628,
            "ttftMs": 530
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r1-qwen-mlx-complex-work-craftbook-author-fanout",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-fanout",
        "repetition": 1
      },
      "trialId": "craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-29T20-21-38-892Z-yv8n",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-29T20-21-38-892Z-yv8n/recording/chat-events.jsonl",
      "recordingSha256": "e8229c1b0b56942a765c1cd3dbf0b8222cdb1c7313cf3958d5f8ad909145d2e1",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r1-qwen-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-29T20-21-38-892Z-yv8n/metrics.json",
      "metricsSha256": "198e0e0b40ea0588d400f3935cca4289dd143c03a98fc59d800c703606c85d34",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 259,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 35,
            "rx": "2026-10-29T20:22:10.393Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 773,
            "ttftMs": 29938
          },
          {
            "line": 4045,
            "rx": "2026-10-29T20:35:45.702Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 221,
            "ttftMs": 2366
          },
          {
            "line": 8342,
            "rx": "2026-10-29T20:49:46.627Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 259,
            "ttftMs": 1415
          },
          {
            "line": 10458,
            "rx": "2026-10-29T20:56:45.753Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 241,
            "ttftMs": 8695
          },
          {
            "line": 10544,
            "rx": "2026-10-29T20:57:06.065Z",
            "sessionId": "eeb197a8-aa5c-41bd-ac42-2a3948909dd0",
            "source": "first-token-detail",
            "tokensPerSec": 186,
            "ttftMs": 69883
          },
          {
            "line": 10887,
            "rx": "2026-10-29T20:58:17.532Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 185,
            "ttftMs": 2675
          },
          {
            "line": 11430,
            "rx": "2026-10-29T21:00:22.446Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 223,
            "ttftMs": 5440
          },
          {
            "line": 13626,
            "rx": "2026-10-29T21:08:50.167Z",
            "sessionId": "eeb197a8-aa5c-41bd-ac42-2a3948909dd0",
            "source": "first-token-detail",
            "tokensPerSec": 462,
            "ttftMs": 138763
          },
          {
            "line": 23511,
            "rx": "2026-10-29T21:38:06.583Z",
            "sessionId": "ffd39342-03a0-4b6c-99ff-9ec759800bdc",
            "source": "first-token-detail",
            "tokensPerSec": 483,
            "ttftMs": 64517
          },
          {
            "line": 23534,
            "rx": "2026-10-29T21:38:17.820Z",
            "sessionId": "aad2117b-c058-4e67-afd4-97213066f8ae",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 23519
          },
          {
            "line": 23916,
            "rx": "2026-10-29T21:39:24.361Z",
            "sessionId": "78a55ded-bbd9-44e6-9970-292d1b02fac5",
            "source": "first-token-detail",
            "tokensPerSec": 143,
            "ttftMs": 4938
          },
          {
            "line": 24264,
            "rx": "2026-10-29T21:40:34.463Z",
            "sessionId": "5f66da49-fe8e-4c8c-ae49-4683205d8f66",
            "source": "first-token-detail",
            "tokensPerSec": 469,
            "ttftMs": 32316
          },
          {
            "line": 24933,
            "rx": "2026-10-29T21:42:31.693Z",
            "sessionId": "aad2117b-c058-4e67-afd4-97213066f8ae",
            "source": "first-token-detail",
            "tokensPerSec": 416,
            "ttftMs": 36602
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-tictactoe",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "tictactoe",
        "repetition": 2
      },
      "trialId": "tictactoe-mlx-gemma4-12b-q4-2026-10-29T22-17-16-394Z-jaso",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-10-29T22-17-16-394Z-jaso/recording/chat-events.jsonl",
      "recordingSha256": "f170ae4d4a4f93f0da93bd26ccb99ab89e0449e2a1d033b3df5e9623d3971110",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-10-29T22-17-16-394Z-jaso/metrics.json",
      "metricsSha256": "7b0d565bc18f7a275fae61ce9c4d50819f4015a92476928f79492839c4122151",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1393,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 23,
            "rx": "2026-10-29T22:17:32.955Z",
            "sessionId": "82a7e574-3dd4-47d4-97e2-54d0058a9c74",
            "source": "first-token-detail",
            "tokensPerSec": 1176,
            "ttftMs": 14871
          },
          {
            "line": 110,
            "rx": "2026-10-29T22:17:51.844Z",
            "sessionId": "d80e50ed-6f6a-4e17-a387-719197a64002",
            "source": "first-token-detail",
            "tokensPerSec": 1681,
            "ttftMs": 3962
          },
          {
            "line": 133,
            "rx": "2026-10-29T22:18:02.408Z",
            "sessionId": "d80e50ed-6f6a-4e17-a387-719197a64002",
            "source": "first-token-detail",
            "tokensPerSec": 1393,
            "ttftMs": 7097
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-tictactoe",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "tictactoe",
        "repetition": 2
      },
      "trialId": "tictactoe-mlx-qwen3-8-27b-q4-2026-10-29T22-21-06-782Z-1qtd",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-10-29T22-21-06-782Z-1qtd/recording/chat-events.jsonl",
      "recordingSha256": "b9b98699c190d7a7be267ea33c86101ceef048071b5fecc91e121d2d753a4e1d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-10-29T22-21-06-782Z-1qtd/metrics.json",
      "metricsSha256": "57d2e630a73b881be7fbbc09f85065c03542744371553661a72e3796055ecba0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 805.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-29T22:21:30.089Z",
            "sessionId": "373a60bb-6d80-43ba-92dd-541f52baa9d7",
            "source": "first-token-detail",
            "tokensPerSec": 784,
            "ttftMs": 21620
          },
          {
            "line": 108,
            "rx": "2026-10-29T22:21:49.912Z",
            "sessionId": "3c9afae1-bd84-43c0-82a0-8e66e0635426",
            "source": "first-token-detail",
            "tokensPerSec": 827,
            "ttftMs": 6641
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-petshop",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "petshop",
        "repetition": 2
      },
      "trialId": "petshop-mlx-gemma4-12b-q4-2026-10-29T22-23-14-365Z-yrrm",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-10-29T22-23-14-365Z-yrrm/recording/chat-events.jsonl",
      "recordingSha256": "866a3691787c7159d1dd6c5624abad104dc2515ad7173df7f79937721dd198df",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-10-29T22-23-14-365Z-yrrm/metrics.json",
      "metricsSha256": "857866284b5a51dfa93d91283d17e01ddb3f34f042d98a09c8fb8b2e8a6da264",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 637,
        "sampleCount": 19,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 19,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 23,
            "rx": "2026-10-29T22:23:30.830Z",
            "sessionId": "481c6c2c-2aef-4da5-925d-de433e5f0f78",
            "source": "first-token-detail",
            "tokensPerSec": 1190,
            "ttftMs": 14784
          },
          {
            "line": 179,
            "rx": "2026-10-29T22:24:08.494Z",
            "sessionId": "8267f590-dffe-4a85-af84-1be03caf17f7",
            "source": "first-token-detail",
            "tokensPerSec": 1205,
            "ttftMs": 12614
          },
          {
            "line": 299,
            "rx": "2026-10-29T22:24:31.455Z",
            "sessionId": "4393ac3f-050b-4f37-85ee-a4a03ab2c8af",
            "source": "first-token-detail",
            "tokensPerSec": 1160,
            "ttftMs": 3371
          },
          {
            "line": 446,
            "rx": "2026-10-29T22:25:17.197Z",
            "sessionId": "3bd439db-e5e1-4dc4-b4e6-1de5aded38dd",
            "source": "first-token-detail",
            "tokensPerSec": 1177,
            "ttftMs": 10476
          },
          {
            "line": 702,
            "rx": "2026-10-29T22:25:59.332Z",
            "sessionId": "8267f590-dffe-4a85-af84-1be03caf17f7",
            "source": "first-token-detail",
            "tokensPerSec": 246,
            "ttftMs": 54820
          },
          {
            "line": 808,
            "rx": "2026-10-29T22:26:17.555Z",
            "sessionId": "8267f590-dffe-4a85-af84-1be03caf17f7",
            "source": "first-token-detail",
            "tokensPerSec": 614,
            "ttftMs": 780
          },
          {
            "line": 1001,
            "rx": "2026-10-29T22:26:46.841Z",
            "sessionId": "4393ac3f-050b-4f37-85ee-a4a03ab2c8af",
            "source": "first-token-detail",
            "tokensPerSec": 191,
            "ttftMs": 34972
          },
          {
            "line": 1191,
            "rx": "2026-10-29T22:27:43.199Z",
            "sessionId": "3bd439db-e5e1-4dc4-b4e6-1de5aded38dd",
            "source": "first-token-detail",
            "tokensPerSec": 822,
            "ttftMs": 1590
          },
          {
            "line": 1551,
            "rx": "2026-10-29T22:28:34.495Z",
            "sessionId": "fd61f77d-a1d2-4bfd-a340-0e982f28611d",
            "source": "first-token-detail",
            "tokensPerSec": 89,
            "ttftMs": 56626
          },
          {
            "line": 1741,
            "rx": "2026-10-29T22:29:32.144Z",
            "sessionId": "4393ac3f-050b-4f37-85ee-a4a03ab2c8af",
            "source": "first-token-detail",
            "tokensPerSec": 875,
            "ttftMs": 8408
          },
          {
            "line": 1931,
            "rx": "2026-10-29T22:30:27.409Z",
            "sessionId": "8267f590-dffe-4a85-af84-1be03caf17f7",
            "source": "first-token-detail",
            "tokensPerSec": 481,
            "ttftMs": 880
          },
          {
            "line": 2346,
            "rx": "2026-10-29T22:31:27.937Z",
            "sessionId": "fd61f77d-a1d2-4bfd-a340-0e982f28611d",
            "source": "first-token-detail",
            "tokensPerSec": 497,
            "ttftMs": 1316
          },
          {
            "line": 2538,
            "rx": "2026-10-29T22:32:17.236Z",
            "sessionId": "3bd439db-e5e1-4dc4-b4e6-1de5aded38dd",
            "source": "first-token-detail",
            "tokensPerSec": 495,
            "ttftMs": 839
          },
          {
            "line": 2808,
            "rx": "2026-10-29T22:33:02.255Z",
            "sessionId": "4393ac3f-050b-4f37-85ee-a4a03ab2c8af",
            "source": "first-token-detail",
            "tokensPerSec": 948,
            "ttftMs": 10468
          },
          {
            "line": 2960,
            "rx": "2026-10-29T22:33:42.616Z",
            "sessionId": "8267f590-dffe-4a85-af84-1be03caf17f7",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 505
          },
          {
            "line": 3421,
            "rx": "2026-10-29T22:34:48.463Z",
            "sessionId": "fd61f77d-a1d2-4bfd-a340-0e982f28611d",
            "source": "first-token-detail",
            "tokensPerSec": 637,
            "ttftMs": 1035
          },
          {
            "line": 3597,
            "rx": "2026-10-29T22:35:35.969Z",
            "sessionId": "3bd439db-e5e1-4dc4-b4e6-1de5aded38dd",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 502
          },
          {
            "line": 3708,
            "rx": "2026-10-29T22:36:01.286Z",
            "sessionId": "4393ac3f-050b-4f37-85ee-a4a03ab2c8af",
            "source": "first-token-detail",
            "tokensPerSec": 963,
            "ttftMs": 11637
          },
          {
            "line": 3888,
            "rx": "2026-10-29T22:36:59.204Z",
            "sessionId": "fd61f77d-a1d2-4bfd-a340-0e982f28611d",
            "source": "first-token-detail",
            "tokensPerSec": 641,
            "ttftMs": 1195
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-petshop",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "petshop",
        "repetition": 2
      },
      "trialId": "petshop-mlx-qwen3-8-27b-q4-2026-10-29T22-40-37-084Z-s09e",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-10-29T22-40-37-084Z-s09e/recording/chat-events.jsonl",
      "recordingSha256": "6eddc24594c3a648a60a3b5e907df252894e1baba8d68e3ac2437c4a7f18fd34",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-10-29T22-40-37-084Z-s09e/metrics.json",
      "metricsSha256": "cdd475c3d5aed3a8533c56051cb937c9128ea1d8e4d17b59a6ff54c7e0e301a5",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 397,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-29T22:41:05.853Z",
            "sessionId": "56d7f858-9795-4992-84b4-0f30d268ea52",
            "source": "first-token-detail",
            "tokensPerSec": 577,
            "ttftMs": 27075
          },
          {
            "line": 157,
            "rx": "2026-10-29T22:41:26.094Z",
            "sessionId": "56d7f858-9795-4992-84b4-0f30d268ea52",
            "source": "first-token-detail",
            "tokensPerSec": 320,
            "ttftMs": 994
          },
          {
            "line": 211,
            "rx": "2026-10-29T22:41:34.983Z",
            "sessionId": "56d7f858-9795-4992-84b4-0f30d268ea52",
            "source": "first-token-detail",
            "tokensPerSec": 305,
            "ttftMs": 994
          },
          {
            "line": 329,
            "rx": "2026-10-29T22:42:15.149Z",
            "sessionId": "267e32f5-d75a-4207-b80c-0dcbf77cad80",
            "source": "first-token-detail",
            "tokensPerSec": 576,
            "ttftMs": 24232
          },
          {
            "line": 1752,
            "rx": "2026-10-29T22:46:33.772Z",
            "sessionId": "8a78bac7-5f8e-4ae2-aacf-e84a0daaf386",
            "source": "first-token-detail",
            "tokensPerSec": 132,
            "ttftMs": 255130
          },
          {
            "line": 1788,
            "rx": "2026-10-29T22:46:58.674Z",
            "sessionId": "307446ba-a4a1-4f9f-9649-dd2af560e228",
            "source": "first-token-detail",
            "tokensPerSec": 474,
            "ttftMs": 25759
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-tankcombat",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "tankcombat",
        "repetition": 2
      },
      "trialId": "tankcombat-mlx-qwen3-8-27b-q4-2026-10-29T23-12-19-097Z-vvpm",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-10-29T23-12-19-097Z-vvpm/recording/chat-events.jsonl",
      "recordingSha256": "48f16e3f083843a00363c04aff2212e762eb30ee2da93832d7efe97586d7fa2e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-10-29T23-12-19-097Z-vvpm/metrics.json",
      "metricsSha256": "3a4e8ad3a1e8c70e48259a65a16287a789f19e9168899745a08d3d16e01ec403",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 775,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-29T23:12:42.015Z",
            "sessionId": "ab644d90-b969-45ad-96a9-1d2b5c25b264",
            "source": "first-token-detail",
            "tokensPerSec": 820,
            "ttftMs": 21242
          },
          {
            "line": 128,
            "rx": "2026-10-29T23:13:03.025Z",
            "sessionId": "e6047ffc-352d-4831-b360-dbc9c3a3b8f6",
            "source": "first-token-detail",
            "tokensPerSec": 730,
            "ttftMs": 7331
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-tankcombat",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "tankcombat",
        "repetition": 2
      },
      "trialId": "tankcombat-mlx-gemma4-12b-q4-2026-10-29T23-19-31-332Z-4q3x",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-10-29T23-19-31-332Z-4q3x/recording/chat-events.jsonl",
      "recordingSha256": "8c0ca0c77f74e23d1a024ae5154cb7a91a756d1b1445f4057da5df967f8446bf",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-10-29T23-19-31-332Z-4q3x/metrics.json",
      "metricsSha256": "1f625e8927dd9bf2f692cc52dba4c3e7acce146609a2cc5d1adf03cf08d5fd7d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1184.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-10-29T23:19:49.513Z",
            "sessionId": "b2b1674d-bf7b-4638-8fbe-4a663f7d0bcf",
            "source": "first-token-detail",
            "tokensPerSec": 1010,
            "ttftMs": 16485
          },
          {
            "line": 185,
            "rx": "2026-10-29T23:20:22.478Z",
            "sessionId": "71ed5335-dcbc-4d0d-bea5-3ecbbaf1dd7d",
            "source": "first-token-detail",
            "tokensPerSec": 1359,
            "ttftMs": 4563
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-schema-migration",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "schema-migration",
        "repetition": 2
      },
      "trialId": "schema-migration-mlx-qwen3-8-27b-q4-2026-10-29T23-22-10-027Z-woe1",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-10-29T23-22-10-027Z-woe1/recording/chat-events.jsonl",
      "recordingSha256": "85b68ed1992521a977efa92b9cd8431ae506bf8c3bd46baf069c2c29455e9696",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-10-29T23-22-10-027Z-woe1/metrics.json",
      "metricsSha256": "18f673f6ea4bdd69c2a837cfed9106c2326a6ab98d6f06bfd0bef38da311cc82",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 657,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-29T23:22:25.241Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 564,
            "ttftMs": 13564
          },
          {
            "line": 74,
            "rx": "2026-10-29T23:22:35.137Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 557,
            "ttftMs": 4827
          },
          {
            "line": 107,
            "rx": "2026-10-29T23:22:47.730Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 657,
            "ttftMs": 5914
          },
          {
            "line": 176,
            "rx": "2026-10-29T23:23:10.021Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 748,
            "ttftMs": 3594
          },
          {
            "line": 246,
            "rx": "2026-10-29T23:23:31.394Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 674,
            "ttftMs": 4887
          },
          {
            "line": 431,
            "rx": "2026-10-29T23:24:17.254Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 711,
            "ttftMs": 5391
          },
          {
            "line": 659,
            "rx": "2026-10-29T23:25:14.726Z",
            "sessionId": "b7285b93-cd4e-4612-8ea5-21c5120248a1",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 12507
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-schema-migration",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "schema-migration",
        "repetition": 2
      },
      "trialId": "schema-migration-mlx-gemma4-12b-q4-2026-10-29T23-30-55-714Z-au6z",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-10-29T23-30-55-714Z-au6z/recording/chat-events.jsonl",
      "recordingSha256": "2e0f77d1d6a68ef42f70dbefce4c5a74ffe5b21402faf4133b820615fd0835db",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-10-29T23-30-55-714Z-au6z/metrics.json",
      "metricsSha256": "e3099554c322caee603767cb0f96a542eefdba78795f9f71e3c7924f44e19084",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 925.5,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-29T23:31:09.208Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 915,
            "ttftMs": 11833
          },
          {
            "line": 133,
            "rx": "2026-10-29T23:31:38.313Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 920,
            "ttftMs": 5891
          },
          {
            "line": 159,
            "rx": "2026-10-29T23:31:52.691Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 1304,
            "ttftMs": 5101
          },
          {
            "line": 188,
            "rx": "2026-10-29T23:32:03.072Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 1384,
            "ttftMs": 873
          },
          {
            "line": 217,
            "rx": "2026-10-29T23:32:24.903Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 1037,
            "ttftMs": 7625
          },
          {
            "line": 251,
            "rx": "2026-10-29T23:32:37.995Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 1152,
            "ttftMs": 630
          },
          {
            "line": 303,
            "rx": "2026-10-29T23:33:16.219Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 931,
            "ttftMs": 11861
          },
          {
            "line": 474,
            "rx": "2026-10-29T23:33:48.828Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 444,
            "ttftMs": 430
          },
          {
            "line": 725,
            "rx": "2026-10-29T23:34:39.792Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 419,
            "ttftMs": 472
          },
          {
            "line": 928,
            "rx": "2026-10-29T23:35:21.515Z",
            "sessionId": "7f02136d-119c-4cd3-b19b-7b399087794f",
            "source": "first-token-detail",
            "tokensPerSec": 900,
            "ttftMs": 11856
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-failing-tests-spec",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "failing-tests-spec",
        "repetition": 2
      },
      "trialId": "failing-tests-spec-mlx-gemma4-12b-q4-2026-10-29T23-40-45-443Z-x030",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-10-29T23-40-45-443Z-x030/recording/chat-events.jsonl",
      "recordingSha256": "86647bbb2025a58cf300012e9fd12dae17940ce924924c513c06e19ade866704",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-10-29T23-40-45-443Z-x030/metrics.json",
      "metricsSha256": "26001ff61e8333c1daacfacee85c9ef0771e1a3655d59f209ce8a7b1919a2e40",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1146,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-29T23:40:57.790Z",
            "sessionId": "90aa3026-cb89-4044-9e9b-63d709767adf",
            "source": "first-token-detail",
            "tokensPerSec": 1035,
            "ttftMs": 10705
          },
          {
            "line": 212,
            "rx": "2026-10-29T23:42:14.137Z",
            "sessionId": "90aa3026-cb89-4044-9e9b-63d709767adf",
            "source": "first-token-detail",
            "tokensPerSec": 1335,
            "ttftMs": 6254
          },
          {
            "line": 328,
            "rx": "2026-10-29T23:43:14.705Z",
            "sessionId": "90aa3026-cb89-4044-9e9b-63d709767adf",
            "source": "first-token-detail",
            "tokensPerSec": 1015,
            "ttftMs": 11578
          },
          {
            "line": 468,
            "rx": "2026-10-29T23:44:24.962Z",
            "sessionId": "90aa3026-cb89-4044-9e9b-63d709767adf",
            "source": "first-token-detail",
            "tokensPerSec": 1257,
            "ttftMs": 936
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-failing-tests-spec",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "failing-tests-spec",
        "repetition": 2
      },
      "trialId": "failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-29T23-46-07-815Z-uxrq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-29T23-46-07-815Z-uxrq/recording/chat-events.jsonl",
      "recordingSha256": "78301a3305d8a015dd49713e21b364153bb76833d8cb20fdf0e9e44b08488537",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-10-29T23-46-07-815Z-uxrq/metrics.json",
      "metricsSha256": "d6d0154f180ee1e61e6437fef02a78df511230f452708c23a6a1d3d5b6cb0e7c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 516,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-29T23:46:23.055Z",
            "sessionId": "27a96fda-01d0-4a5c-8165-51cd7e3739c2",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 13570
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-symptom-debug",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "symptom-debug",
        "repetition": 2
      },
      "trialId": "symptom-debug-mlx-gemma4-12b-q4-2026-10-29T23-47-13-425Z-2hng",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-10-29T23-47-13-425Z-2hng/recording/chat-events.jsonl",
      "recordingSha256": "2d5d56a7564b3d747ab314628ad556fcaf48f551e34ab8d96da443a12ac4f757",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-10-29T23-47-13-425Z-2hng/metrics.json",
      "metricsSha256": "f5add75e99be7dbdd30fe39ece7ea485224986cb97b1e4880d1cacb204a3eddb",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 749.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-29T23:47:25.558Z",
            "sessionId": "4e07aecf-e1ce-4617-8567-f9367c19d269",
            "source": "first-token-detail",
            "tokensPerSec": 1193,
            "ttftMs": 10498
          },
          {
            "line": 1426,
            "rx": "2026-10-29T23:51:13.781Z",
            "sessionId": "4e07aecf-e1ce-4617-8567-f9367c19d269",
            "source": "first-token-detail",
            "tokensPerSec": 306,
            "ttftMs": 485
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-symptom-debug",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "symptom-debug",
        "repetition": 2
      },
      "trialId": "symptom-debug-mlx-qwen3-8-27b-q4-2026-10-29T23-56-42-218Z-a631",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-10-29T23-56-42-218Z-a631/recording/chat-events.jsonl",
      "recordingSha256": "1be2de597be5dd8040ea8cbfa88e97221180539cb33454db824f43c98f65fcd9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-10-29T23-56-42-218Z-a631/metrics.json",
      "metricsSha256": "9f2fd3f9524f0db721cabd46a85941f21dc1f196542e1b5267435ebaac93bbd5",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 512,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-29T23:56:58.472Z",
            "sessionId": "bdaae4e5-6299-457c-9a77-ada12b19aa48",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 14603
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-data-wrangle",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "data-wrangle",
        "repetition": 2
      },
      "trialId": "data-wrangle-mlx-qwen3-8-27b-q4-2026-10-30T00-00-20-916Z-fib5",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-10-30T00-00-20-916Z-fib5/recording/chat-events.jsonl",
      "recordingSha256": "dae05e3bdf1f835d90b08779ed4fdb1ad761fe49226304c92c227691f60aa310",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-10-30T00-00-20-916Z-fib5/metrics.json",
      "metricsSha256": "3c5f04af9d37f73bffa2128e772f4619863dfe6581ece97c82622f1b845bb11d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 612,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T00:00:36.332Z",
            "sessionId": "c1b3f42b-c9f4-4d01-8f98-4826d84270ba",
            "source": "first-token-detail",
            "tokensPerSec": 612,
            "ttftMs": 13763
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-data-wrangle",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "data-wrangle",
        "repetition": 2
      },
      "trialId": "data-wrangle-mlx-gemma4-12b-q4-2026-10-30T00-06-40-900Z-cjrj",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-10-30T00-06-40-900Z-cjrj/recording/chat-events.jsonl",
      "recordingSha256": "6d3db81e5392c2200b9bb2b21fe7207bf1a75c3e94aabf9ef32a97301a1771e9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-10-30T00-06-40-900Z-cjrj/metrics.json",
      "metricsSha256": "ccae58593d8a01b756e5d81ce993937ba6b9b3ea607ec597659409c57ad7d878",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 983,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T00:06:54.760Z",
            "sessionId": "a259f926-3f03-4213-ac01-3ae00b7470c7",
            "source": "first-token-detail",
            "tokensPerSec": 881,
            "ttftMs": 12227
          },
          {
            "line": 1880,
            "rx": "2026-10-30T00:12:20.688Z",
            "sessionId": "a259f926-3f03-4213-ac01-3ae00b7470c7",
            "source": "first-token-detail",
            "tokensPerSec": 307,
            "ttftMs": 485
          },
          {
            "line": 2279,
            "rx": "2026-10-30T00:13:51.033Z",
            "sessionId": "a259f926-3f03-4213-ac01-3ae00b7470c7",
            "source": "first-token-detail",
            "tokensPerSec": 983,
            "ttftMs": 11226
          },
          {
            "line": 2388,
            "rx": "2026-10-30T00:14:26.266Z",
            "sessionId": "a259f926-3f03-4213-ac01-3ae00b7470c7",
            "source": "first-token-detail",
            "tokensPerSec": 1015,
            "ttftMs": 1872
          },
          {
            "line": 2517,
            "rx": "2026-10-30T00:15:39.405Z",
            "sessionId": "a259f926-3f03-4213-ac01-3ae00b7470c7",
            "source": "first-token-detail",
            "tokensPerSec": 985,
            "ttftMs": 19228
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-incident-postmortem",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "incident-postmortem",
        "repetition": 2
      },
      "trialId": "incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-30T00-16-38-042Z-e5em",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-30T00-16-38-042Z-e5em/recording/chat-events.jsonl",
      "recordingSha256": "fb07e0e7b46c44e0a7be58911ba11071e9a11737da00bfcd67cbb86f4afd2ecc",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-10-30T00-16-38-042Z-e5em/metrics.json",
      "metricsSha256": "c44db17d29d69bf4f08ec00a1f3418bb08ce5d43f1985478f5b6a43d3c799530",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 507.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-30T00:17:07.729Z",
            "sessionId": "59b82ba3-db1f-4651-be3b-65b606c1a014",
            "source": "first-token-detail",
            "tokensPerSec": 527,
            "ttftMs": 28040
          },
          {
            "line": 4873,
            "rx": "2026-10-30T00:31:23.633Z",
            "sessionId": "59b82ba3-db1f-4651-be3b-65b606c1a014",
            "source": "first-token-detail",
            "tokensPerSec": 488,
            "ttftMs": 22053
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-incident-postmortem",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "incident-postmortem",
        "repetition": 2
      },
      "trialId": "incident-postmortem-mlx-gemma4-12b-q4-2026-10-30T00-41-35-480Z-a2in",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-10-30T00-41-35-480Z-a2in/recording/chat-events.jsonl",
      "recordingSha256": "8a2c37f35ffc9633f8f3af993d0a75074de3d1d9742c13893028478a3592807c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-10-30T00-41-35-480Z-a2in/metrics.json",
      "metricsSha256": "9e274a8a92d53c99e3f1027819523446e9dbd949576d3c9cbe32eccb5aebea1b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1090,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-30T00:41:53.785Z",
            "sessionId": "c1c9b85b-dccb-441b-a72e-a00b09668c63",
            "source": "first-token-detail",
            "tokensPerSec": 989,
            "ttftMs": 16668
          },
          {
            "line": 459,
            "rx": "2026-10-30T00:43:55.113Z",
            "sessionId": "c1c9b85b-dccb-441b-a72e-a00b09668c63",
            "source": "first-token-detail",
            "tokensPerSec": 1090,
            "ttftMs": 12416
          },
          {
            "line": 492,
            "rx": "2026-10-30T00:44:03.592Z",
            "sessionId": "c1c9b85b-dccb-441b-a72e-a00b09668c63",
            "source": "first-token-detail",
            "tokensPerSec": 878,
            "ttftMs": 1401
          },
          {
            "line": 525,
            "rx": "2026-10-30T00:44:13.296Z",
            "sessionId": "c1c9b85b-dccb-441b-a72e-a00b09668c63",
            "source": "first-token-detail",
            "tokensPerSec": 1182,
            "ttftMs": 1052
          },
          {
            "line": 571,
            "rx": "2026-10-30T00:44:28.323Z",
            "sessionId": "c1c9b85b-dccb-441b-a72e-a00b09668c63",
            "source": "first-token-detail",
            "tokensPerSec": 1231,
            "ttftMs": 1006
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-ops-runbook-anomaly",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "ops-runbook-anomaly",
        "repetition": 2
      },
      "trialId": "ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-30T00-50-27-551Z-iukg",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-30T00-50-27-551Z-iukg/recording/chat-events.jsonl",
      "recordingSha256": "cab3d3b34867c9836d5472c8d254a4704d6c40ff7031d9d301602f599b67f9e6",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-10-30T00-50-27-551Z-iukg/metrics.json",
      "metricsSha256": "394856a33dd8208c0102cbce271550e76ca2d91af08c1f479ff3f8a23943470f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 989.5,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T00:50:42.153Z",
            "sessionId": "a3eba46d-877d-4c0f-b984-dd9eba7cc71b",
            "source": "first-token-detail",
            "tokensPerSec": 1325,
            "ttftMs": 12878
          },
          {
            "line": 1560,
            "rx": "2026-10-30T00:54:49.544Z",
            "sessionId": "a3eba46d-877d-4c0f-b984-dd9eba7cc71b",
            "source": "first-token-detail",
            "tokensPerSec": 208,
            "ttftMs": 440
          },
          {
            "line": 1724,
            "rx": "2026-10-30T00:55:23.171Z",
            "sessionId": "a3eba46d-877d-4c0f-b984-dd9eba7cc71b",
            "source": "first-token-detail",
            "tokensPerSec": 1036,
            "ttftMs": 6242
          },
          {
            "line": 1769,
            "rx": "2026-10-30T00:55:44.896Z",
            "sessionId": "a3eba46d-877d-4c0f-b984-dd9eba7cc71b",
            "source": "first-token-detail",
            "tokensPerSec": 943,
            "ttftMs": 6995
          },
          {
            "line": 1856,
            "rx": "2026-10-30T00:56:22.894Z",
            "sessionId": "a3eba46d-877d-4c0f-b984-dd9eba7cc71b",
            "source": "first-token-detail",
            "tokensPerSec": 929,
            "ttftMs": 8644
          },
          {
            "line": 1917,
            "rx": "2026-10-30T00:57:03.891Z",
            "sessionId": "a3eba46d-877d-4c0f-b984-dd9eba7cc71b",
            "source": "first-token-detail",
            "tokensPerSec": 1114,
            "ttftMs": 8429
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-ops-runbook-anomaly",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "ops-runbook-anomaly",
        "repetition": 2
      },
      "trialId": "ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-30T00-59-49-741Z-neo6",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-30T00-59-49-741Z-neo6/recording/chat-events.jsonl",
      "recordingSha256": "bd97f2f9ca75007fb16e011d23bc2f5e3260815961b4de541608614bda448e80",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-10-30T00-59-49-741Z-neo6/metrics.json",
      "metricsSha256": "903510837b834c2ce612703558d54a66bc4bd88108fa7837b76d0f3c9ba7612e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 558,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T01:00:18.285Z",
            "sessionId": "dd1308cd-3a41-47e1-8fb5-0423e616a179",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 26884
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-plan-and-estimate",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "plan-and-estimate",
        "repetition": 2
      },
      "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-02-14-249Z-r3ve",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-02-14-249Z-r3ve/recording/chat-events.jsonl",
      "recordingSha256": "1415bdb31059ac13102f7966c916dbef482fc69a0e4e040164c644d1c502d628",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-02-14-249Z-r3ve/metrics.json",
      "metricsSha256": "018bac092b2fa4312f0bb2b123231fbb4432e0bdc4a85867f13d0bd1c4f72d9d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1123,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T01:02:26.847Z",
            "sessionId": "64856e0f-91ce-4a9e-beb5-77eb4b1dd208",
            "source": "first-token-detail",
            "tokensPerSec": 1123,
            "ttftMs": 10971
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-plan-and-estimate",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "plan-and-estimate",
        "repetition": 2
      },
      "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-04-14-071Z-wx28",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-04-14-071Z-wx28/recording/chat-events.jsonl",
      "recordingSha256": "00d2348bd9b86d02f0f74e3cd9f2036f294defa350e766b5cdbe0e8af3d293fb",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-04-14-071Z-wx28/metrics.json",
      "metricsSha256": "5582c4502c43a7d8cdb998a48bdc1783f9f287f580b5f4a51134df3d20358b77",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 612,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T01:04:29.557Z",
            "sessionId": "d4fcfa1e-af52-439c-8089-b8f77ee756e7",
            "source": "first-token-detail",
            "tokensPerSec": 612,
            "ttftMs": 13845
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-core-conflict-synthesis",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "conflict-synthesis",
        "repetition": 2
      },
      "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T01-08-06-141Z-slji",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T01-08-06-141Z-slji/recording/chat-events.jsonl",
      "recordingSha256": "c29c193cd6d29b10ec14f6eab80d66bffce7fa82f5423d165b4428de740de21a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T01-08-06-141Z-slji/metrics.json",
      "metricsSha256": "7075adcabd23249be1d6bba271467830e5fc8dd3fa8016e2a2eb1efc72ec7137",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 615,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T01:08:32.814Z",
            "sessionId": "c16f2a1a-e2aa-4bb1-90bf-812216e26af2",
            "source": "first-token-detail",
            "tokensPerSec": 615,
            "ttftMs": 25026
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-core-conflict-synthesis",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "conflict-synthesis",
        "repetition": 2
      },
      "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T01-15-21-615Z-vhak",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T01-15-21-615Z-vhak/recording/chat-events.jsonl",
      "recordingSha256": "0f8a8396f0843ae2f6c409cba94158c51c287e387a946c34714302e57efb337b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T01-15-21-615Z-vhak/metrics.json",
      "metricsSha256": "5ab0282e400f3e8c2c6bb0d6545289c38be4e5bf26cb2bccbef3b712a0de27f9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1116.5,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-30T01:15:39.470Z",
            "sessionId": "a4c79eb6-70b2-4943-ab87-73d2f7247dcd",
            "source": "first-token-detail",
            "tokensPerSec": 1006,
            "ttftMs": 16218
          },
          {
            "line": 202,
            "rx": "2026-10-30T01:16:18.888Z",
            "sessionId": "a4c79eb6-70b2-4943-ab87-73d2f7247dcd",
            "source": "first-token-detail",
            "tokensPerSec": 1129,
            "ttftMs": 5383
          },
          {
            "line": 333,
            "rx": "2026-10-30T01:17:08.756Z",
            "sessionId": "a4c79eb6-70b2-4943-ab87-73d2f7247dcd",
            "source": "first-token-detail",
            "tokensPerSec": 1329,
            "ttftMs": 5593
          },
          {
            "line": 533,
            "rx": "2026-10-30T01:18:27.339Z",
            "sessionId": "a4c79eb6-70b2-4943-ab87-73d2f7247dcd",
            "source": "first-token-detail",
            "tokensPerSec": 1151,
            "ttftMs": 8904
          },
          {
            "line": 625,
            "rx": "2026-10-30T01:18:55.329Z",
            "sessionId": "a4c79eb6-70b2-4943-ab87-73d2f7247dcd",
            "source": "first-token-detail",
            "tokensPerSec": 1104,
            "ttftMs": 11789
          },
          {
            "line": 680,
            "rx": "2026-10-30T01:19:20.902Z",
            "sessionId": "a4c79eb6-70b2-4943-ab87-73d2f7247dcd",
            "source": "first-token-detail",
            "tokensPerSec": 995,
            "ttftMs": 12257
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-constrained-comms",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "constrained-comms",
        "repetition": 2
      },
      "trialId": "constrained-comms-mlx-qwen3-8-27b-q4-2026-10-30T01-20-16-922Z-1u4u",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-10-30T01-20-16-922Z-1u4u/recording/chat-events.jsonl",
      "recordingSha256": "f4c491a70510053ce522910bfc1f3198e056a2a373e1005db5b5e693d6ef3995",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-10-30T01-20-16-922Z-1u4u/metrics.json",
      "metricsSha256": "17285d806dbeb66176e72772198e47235d4db195cfd2125a44ace806e2de088c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 611,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T01:20:46.498Z",
            "sessionId": "6400ac8b-5689-425e-bf2c-3e7432f40171",
            "source": "first-token-detail",
            "tokensPerSec": 522,
            "ttftMs": 27930
          },
          {
            "line": 96,
            "rx": "2026-10-30T01:21:11.234Z",
            "sessionId": "6400ac8b-5689-425e-bf2c-3e7432f40171",
            "source": "first-token-detail",
            "tokensPerSec": 611,
            "ttftMs": 7487
          },
          {
            "line": 150,
            "rx": "2026-10-30T01:21:32.556Z",
            "sessionId": "6400ac8b-5689-425e-bf2c-3e7432f40171",
            "source": "first-token-detail",
            "tokensPerSec": 713,
            "ttftMs": 4194
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-constrained-comms",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "constrained-comms",
        "repetition": 2
      },
      "trialId": "constrained-comms-mlx-gemma4-12b-q4-2026-10-30T01-23-52-517Z-d5qq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-10-30T01-23-52-517Z-d5qq/recording/chat-events.jsonl",
      "recordingSha256": "7563d778988b658db4a9034ebc33373d415f3b27b7ea9174e45843aa25945199",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-10-30T01-23-52-517Z-d5qq/metrics.json",
      "metricsSha256": "b729b5b4249fd849ed789d65f7c24683baf1cbd383baf5b3cecd1d5d8709c3f7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1190.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-30T01:24:08.969Z",
            "sessionId": "1d95d62d-b2c5-4ff9-883c-a5b96a0d3851",
            "source": "first-token-detail",
            "tokensPerSec": 1174,
            "ttftMs": 14792
          },
          {
            "line": 122,
            "rx": "2026-10-30T01:24:34.170Z",
            "sessionId": "1d95d62d-b2c5-4ff9-883c-a5b96a0d3851",
            "source": "first-token-detail",
            "tokensPerSec": 1207,
            "ttftMs": 4852
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-records-intake",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "records-intake",
        "repetition": 2
      },
      "trialId": "records-intake-mlx-gemma4-12b-q4-2026-10-30T01-29-09-739Z-druf",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-10-30T01-29-09-739Z-druf/recording/chat-events.jsonl",
      "recordingSha256": "55823794b617504ac2be05904b8a33a061908bb3c0dab284770cd77ee06b3c07",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-10-30T01-29-09-739Z-druf/metrics.json",
      "metricsSha256": "dd59928b3fb497e5b9abbd9dba64d1c589906b42199c4e22136a001828fe7e71",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1087,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T01:29:22.432Z",
            "sessionId": "3c95d733-d97b-4c41-bfd2-fd81cb0bec80",
            "source": "first-token-detail",
            "tokensPerSec": 1070,
            "ttftMs": 11060
          },
          {
            "line": 436,
            "rx": "2026-10-30T01:31:15.459Z",
            "sessionId": "3c95d733-d97b-4c41-bfd2-fd81cb0bec80",
            "source": "first-token-detail",
            "tokensPerSec": 1104,
            "ttftMs": 8553
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-records-intake",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "records-intake",
        "repetition": 2
      },
      "trialId": "records-intake-mlx-qwen3-8-27b-q4-2026-10-30T01-36-02-730Z-685a",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-10-30T01-36-02-730Z-685a/recording/chat-events.jsonl",
      "recordingSha256": "4cd4717a288ee8bd96b65b5d9282d6f815b5f551a58da6f63e15d5c83e6fdecf",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-10-30T01-36-02-730Z-685a/metrics.json",
      "metricsSha256": "7dddd8ec4e91b2337894e49eade12c796ce63495f232c69f3ffd1c5f84a6209e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 615,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T01:36:17.763Z",
            "sessionId": "395731b8-d470-4827-8415-1b68877e10ac",
            "source": "first-token-detail",
            "tokensPerSec": 615,
            "ttftMs": 13401
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-plan-and-estimate",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "plan-and-estimate",
        "repetition": 2
      },
      "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-37-46-515Z-5dua",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-37-46-515Z-5dua/recording/chat-events.jsonl",
      "recordingSha256": "c1f1d5c10a9e7aee89bd7f84bb1459d9f26bc6cab3340a3212d8348d65923f96",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-10-30T01-37-46-515Z-5dua/metrics.json",
      "metricsSha256": "e5d77a78b11efa7724cc905923a9d9aa3742c43d133361bf6d47c823ee2a62a2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1261,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T01:37:58.606Z",
            "sessionId": "262d7c1d-9ad6-4b7b-9d66-1429f0ef9aa9",
            "source": "first-token-detail",
            "tokensPerSec": 1261,
            "ttftMs": 10469
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-plan-and-estimate",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "plan-and-estimate",
        "repetition": 2
      },
      "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-39-45-725Z-zfdd",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-39-45-725Z-zfdd/recording/chat-events.jsonl",
      "recordingSha256": "a4fa7c20a1b116e03b0b2099ed0c92e9d4a3367f6aae420f416797f653768a72",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-10-30T01-39-45-725Z-zfdd/metrics.json",
      "metricsSha256": "1e11b108f36e0e13c5f7042ad787471937238e5d7d50c931221661c82135a85b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 642,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T01:40:00.872Z",
            "sessionId": "4ef0f451-078f-4e44-89a9-369f2b79777d",
            "source": "first-token-detail",
            "tokensPerSec": 642,
            "ttftMs": 13505
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-meeting-followup",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "meeting-followup",
        "repetition": 2
      },
      "trialId": "meeting-followup-mlx-qwen3-8-27b-q4-2026-10-30T01-46-57-377Z-izhl",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-10-30T01-46-57-377Z-izhl/recording/chat-events.jsonl",
      "recordingSha256": "61fdbe94734d65060a4c61513943e94119e4114cf74ae3af9f1b0da654677084",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-10-30T01-46-57-377Z-izhl/metrics.json",
      "metricsSha256": "fba24155d76dd6f91ca70b268a3d1b90546c658aaf6f8f4c9fb1aa1128ec9bd3",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 722.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T01:47:14.567Z",
            "sessionId": "591d8a1e-4c44-4d4d-b331-0be71efde233",
            "source": "first-token-detail",
            "tokensPerSec": 853,
            "ttftMs": 15558
          },
          {
            "line": 175,
            "rx": "2026-10-30T01:49:08.107Z",
            "sessionId": "591d8a1e-4c44-4d4d-b331-0be71efde233",
            "source": "first-token-detail",
            "tokensPerSec": 592,
            "ttftMs": 3913
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-meeting-followup",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "meeting-followup",
        "repetition": 2
      },
      "trialId": "meeting-followup-mlx-gemma4-12b-q4-2026-10-30T01-52-17-930Z-hf22",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-10-30T01-52-17-930Z-hf22/recording/chat-events.jsonl",
      "recordingSha256": "2c9892e5a6f726cab18c18f3a6726e872db21c68b95ad55d5f6cc30846fd5c15",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-10-30T01-52-17-930Z-hf22/metrics.json",
      "metricsSha256": "c6fda70c633faf2797475eb642e9b4440555bee2aeea6bd9f16e88df38c1bba4",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1921,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-10-30T01:52:29.480Z",
            "sessionId": "a102447a-84d1-41db-9fa5-5b1de82ac8c9",
            "source": "first-token-detail",
            "tokensPerSec": 1925,
            "ttftMs": 9913
          },
          {
            "line": 118,
            "rx": "2026-10-30T01:54:18.715Z",
            "sessionId": "a102447a-84d1-41db-9fa5-5b1de82ac8c9",
            "source": "first-token-detail",
            "tokensPerSec": 1921,
            "ttftMs": 4034
          },
          {
            "line": 611,
            "rx": "2026-10-30T01:55:39.568Z",
            "sessionId": "a102447a-84d1-41db-9fa5-5b1de82ac8c9",
            "source": "first-token-detail",
            "tokensPerSec": 1526,
            "ttftMs": 4598
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-craftbook-spreadsheet-model",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-spreadsheet-model",
        "repetition": 2
      },
      "trialId": "craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-30T01-56-09-353Z-t72x",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-30T01-56-09-353Z-t72x/recording/chat-events.jsonl",
      "recordingSha256": "df23bc1f5b00da693cdbdc6862abb95882d3edcd675bcc2b84e6fe4cf9074274",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-10-30T01-56-09-353Z-t72x/metrics.json",
      "metricsSha256": "3a459e1bcb2287bbe1cd96be2996d6bb46be2cc85ac8e1f5313473823f32b6ae",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 530,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-30T01:56:37.528Z",
            "sessionId": "4c7d1b5d-ec27-4939-8172-9911f27c58a8",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 21669
          },
          {
            "line": 466,
            "rx": "2026-10-30T01:58:11.511Z",
            "sessionId": "cf197a10-05b4-4dc6-b361-02fe8974b995",
            "source": "first-token-detail",
            "tokensPerSec": 559,
            "ttftMs": 15561
          },
          {
            "line": 3123,
            "rx": "2026-10-30T02:06:03.800Z",
            "sessionId": "340883cc-d1d4-48c5-90f3-dcc1564c346e",
            "source": "first-token-detail",
            "tokensPerSec": 493,
            "ttftMs": 17694
          },
          {
            "line": 10250,
            "rx": "2026-10-30T02:27:17.367Z",
            "sessionId": "d3827601-366d-4eb0-a31c-f4f2272e6bb2",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 20758
          },
          {
            "line": 11861,
            "rx": "2026-10-30T02:32:46.344Z",
            "sessionId": "63879e81-683b-4c85-b259-bfaabfb8e175",
            "source": "first-token-detail",
            "tokensPerSec": 530,
            "ttftMs": 19634
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-craftbook-spreadsheet-model",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-spreadsheet-model",
        "repetition": 2
      },
      "trialId": "craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-30T02-45-24-448Z-z5fs",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-30T02-45-24-448Z-z5fs/recording/chat-events.jsonl",
      "recordingSha256": "3f79db678da5fc539d1c42161e65a72facc98fa9093923c56a2ef9660225b708",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-10-30T02-45-24-448Z-z5fs/metrics.json",
      "metricsSha256": "feb2000650e7e5b66112245fa5dc9c07be027cb55168b7c0adafd43bf8f663f9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1196,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T02:45:44.755Z",
            "sessionId": "82d1c8b1-d2d7-4a02-89a5-7a1b06a8e7eb",
            "source": "first-token-detail",
            "tokensPerSec": 1106,
            "ttftMs": 13797
          },
          {
            "line": 172,
            "rx": "2026-10-30T02:46:18.730Z",
            "sessionId": "ca6842af-45bc-4f2c-ac90-a698eabe8287",
            "source": "first-token-detail",
            "tokensPerSec": 1196,
            "ttftMs": 7807
          },
          {
            "line": 384,
            "rx": "2026-10-30T02:47:27.028Z",
            "sessionId": "aab5b556-0100-413e-a086-f9a49a176b86",
            "source": "first-token-detail",
            "tokensPerSec": 1463,
            "ttftMs": 5963
          },
          {
            "line": 585,
            "rx": "2026-10-30T02:48:28.735Z",
            "sessionId": "c34a70ee-6713-4c44-b482-eee9be2c7842",
            "source": "first-token-detail",
            "tokensPerSec": 1246,
            "ttftMs": 7765
          },
          {
            "line": 836,
            "rx": "2026-10-30T02:49:47.630Z",
            "sessionId": "f4d5ad39-d350-49bb-a328-b40c019398c5",
            "source": "first-token-detail",
            "tokensPerSec": 1140,
            "ttftMs": 6626
          },
          {
            "line": 1114,
            "rx": "2026-10-30T02:51:08.738Z",
            "sessionId": "afbb845d-0b69-44ad-ac71-6b29325cc592",
            "source": "first-token-detail",
            "tokensPerSec": 1050,
            "ttftMs": 7744
          },
          {
            "line": 1206,
            "rx": "2026-10-30T02:51:31.467Z",
            "sessionId": "6752f0c9-d34d-441e-b69d-b1611f114aa6",
            "source": "first-token-detail",
            "tokensPerSec": 1347,
            "ttftMs": 3802
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-conflict-synthesis",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "conflict-synthesis",
        "repetition": 2
      },
      "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T03-00-07-646Z-dpj6",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T03-00-07-646Z-dpj6/recording/chat-events.jsonl",
      "recordingSha256": "1a7983c798e8ff9fecaf27c795bd618d2771ff08284eb7e6b64fc9e11e693d3c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-10-30T03-00-07-646Z-dpj6/metrics.json",
      "metricsSha256": "66877f7d35f69bee87d49ba93d43eaa1f9464123028e4628baff27ba1efb106a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1232,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-30T03:00:25.600Z",
            "sessionId": "2c939f1a-d758-47ad-ad32-22417c830ccb",
            "source": "first-token-detail",
            "tokensPerSec": 999,
            "ttftMs": 16277
          },
          {
            "line": 176,
            "rx": "2026-10-30T03:01:03.619Z",
            "sessionId": "2c939f1a-d758-47ad-ad32-22417c830ccb",
            "source": "first-token-detail",
            "tokensPerSec": 1452,
            "ttftMs": 4101
          },
          {
            "line": 273,
            "rx": "2026-10-30T03:01:39.636Z",
            "sessionId": "2c939f1a-d758-47ad-ad32-22417c830ccb",
            "source": "first-token-detail",
            "tokensPerSec": 1232,
            "ttftMs": 5474
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-conflict-synthesis",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "conflict-synthesis",
        "repetition": 2
      },
      "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T03-06-17-524Z-khzv",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T03-06-17-524Z-khzv/recording/chat-events.jsonl",
      "recordingSha256": "050abbd51fc248220d4e6fa995afd80f352ddbaf4688e997a63fa60d36eda072",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-10-30T03-06-17-524Z-khzv/metrics.json",
      "metricsSha256": "3428e48cf8b8705ed92f07b0946cdc39a0cb7253f3d7276f66fa88c721674650",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 574,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-30T03:06:47.736Z",
            "sessionId": "9e09cd27-4e46-4f4e-bf6b-34b82f2df0cc",
            "source": "first-token-detail",
            "tokensPerSec": 515,
            "ttftMs": 28560
          },
          {
            "line": 900,
            "rx": "2026-10-30T03:09:26.906Z",
            "sessionId": "9e09cd27-4e46-4f4e-bf6b-34b82f2df0cc",
            "source": "first-token-detail",
            "tokensPerSec": 599,
            "ttftMs": 7045
          },
          {
            "line": 1327,
            "rx": "2026-10-30T03:11:06.578Z",
            "sessionId": "9e09cd27-4e46-4f4e-bf6b-34b82f2df0cc",
            "source": "first-token-detail",
            "tokensPerSec": 574,
            "ttftMs": 6953
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-docblocks-theme-roundtrip",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "docblocks-theme-roundtrip",
        "repetition": 2
      },
      "trialId": "docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-30T03-12-38-000Z-qua9",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-30T03-12-38-000Z-qua9/recording/chat-events.jsonl",
      "recordingSha256": "ba6edebbe46024f2db3641fc3b64ca46e427baa7b54bc573cf7070f1e836b527",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-10-30T03-12-38-000Z-qua9/metrics.json",
      "metricsSha256": "3e1c42c049281fac62102f4788dd6afec6fd4902300ed251ee88945b76700f01",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1045,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T03:12:51.281Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 1045,
            "ttftMs": 11638
          },
          {
            "line": 192,
            "rx": "2026-10-30T03:14:05.103Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 1245,
            "ttftMs": 541
          },
          {
            "line": 2264,
            "rx": "2026-10-30T03:19:37.154Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 1069,
            "ttftMs": 6306
          },
          {
            "line": 2334,
            "rx": "2026-10-30T03:20:01.625Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 1171,
            "ttftMs": 5688
          },
          {
            "line": 2359,
            "rx": "2026-10-30T03:20:11.606Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 1108,
            "ttftMs": 598
          },
          {
            "line": 2379,
            "rx": "2026-10-30T03:20:16.686Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 984,
            "ttftMs": 651
          },
          {
            "line": 2403,
            "rx": "2026-10-30T03:20:22.156Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 626,
            "ttftMs": 1095
          },
          {
            "line": 2476,
            "rx": "2026-10-30T03:20:56.024Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 982,
            "ttftMs": 9866
          },
          {
            "line": 2514,
            "rx": "2026-10-30T03:21:22.098Z",
            "sessionId": "0f0d751d-1529-48c1-b950-89ec7a89f85c",
            "source": "first-token-detail",
            "tokensPerSec": 969,
            "ttftMs": 10844
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-docblocks-theme-roundtrip",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "docblocks-theme-roundtrip",
        "repetition": 2
      },
      "trialId": "docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-30T03-36-40-892Z-0z2j",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-30T03-36-40-892Z-0z2j/recording/chat-events.jsonl",
      "recordingSha256": "b6f8011fe6514d1b05c448b873be20f8b2b6d46916311a7a0fcc9f3846f157d5",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-10-30T03-36-40-892Z-0z2j/metrics.json",
      "metricsSha256": "77d50b069919d40cd56d1cbcce2431544c6c1588ecae999011936579a37f05e9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 634,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T03:36:54.569Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 934,
            "ttftMs": 12155
          },
          {
            "line": 115,
            "rx": "2026-10-30T03:37:07.917Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 550,
            "ttftMs": 1166
          },
          {
            "line": 222,
            "rx": "2026-10-30T03:37:33.386Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 776,
            "ttftMs": 6259
          },
          {
            "line": 389,
            "rx": "2026-10-30T03:38:13.310Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 634,
            "ttftMs": 6048
          },
          {
            "line": 439,
            "rx": "2026-10-30T03:38:34.766Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 649,
            "ttftMs": 7430
          },
          {
            "line": 489,
            "rx": "2026-10-30T03:39:01.929Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 624,
            "ttftMs": 9506
          },
          {
            "line": 576,
            "rx": "2026-10-30T03:39:38.632Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 644,
            "ttftMs": 11077
          },
          {
            "line": 641,
            "rx": "2026-10-30T03:40:17.467Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 569,
            "ttftMs": 14780
          },
          {
            "line": 709,
            "rx": "2026-10-30T03:41:03.759Z",
            "sessionId": "3606ed2d-c843-4273-99d1-9d669202a1dc",
            "source": "first-token-detail",
            "tokensPerSec": 469,
            "ttftMs": 20952
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-craftbook-powerpoint-deck",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-powerpoint-deck",
        "repetition": 2
      },
      "trialId": "craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-30T03-52-33-429Z-u0rh",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-30T03-52-33-429Z-u0rh/recording/chat-events.jsonl",
      "recordingSha256": "60a123924d3262db8cf1585e4c7875b292ee2fb2188f41735b6e2ddae38110b5",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-10-30T03-52-33-429Z-u0rh/metrics.json",
      "metricsSha256": "98a4635d4b6f6999a24817eabcfadaf79fcb83ab7169b95db4c2e0e82ee1ddc2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 522,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-30T03:53:03.097Z",
            "sessionId": "972c66d3-bba3-48d2-a865-41cf8cf65ca2",
            "source": "first-token-detail",
            "tokensPerSec": 572,
            "ttftMs": 23117
          },
          {
            "line": 1106,
            "rx": "2026-10-30T03:56:20.624Z",
            "sessionId": "931f01c4-8167-4c11-a4d4-28000e1490e2",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 15529
          },
          {
            "line": 2161,
            "rx": "2026-10-30T03:59:46.994Z",
            "sessionId": "f131fadd-1ff1-4ee3-92c0-6708b6580f6c",
            "source": "first-token-detail",
            "tokensPerSec": 522,
            "ttftMs": 16824
          },
          {
            "line": 2666,
            "rx": "2026-10-30T04:01:36.968Z",
            "sessionId": "992e117e-a9fb-42fe-869f-bc61f4196745",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 16777
          },
          {
            "line": 5760,
            "rx": "2026-10-30T04:11:22.038Z",
            "sessionId": "9ef128af-2bd7-47d8-bd67-405a3e9fb57c",
            "source": "first-token-detail",
            "tokensPerSec": 542,
            "ttftMs": 16767
          },
          {
            "line": 6103,
            "rx": "2026-10-30T04:13:36.613Z",
            "sessionId": "afe3028c-2edf-430e-b6b2-9a74deb2be14",
            "source": "first-token-detail",
            "tokensPerSec": 677,
            "ttftMs": 6163
          },
          {
            "line": 7556,
            "rx": "2026-10-30T04:19:27.523Z",
            "sessionId": "b1348b8b-6e4f-4b16-8f68-5f03b3ab100f",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 51383
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-craftbook-powerpoint-deck",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-powerpoint-deck",
        "repetition": 2
      },
      "trialId": "craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-30T04-41-50-457Z-1glo",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-30T04-41-50-457Z-1glo/recording/chat-events.jsonl",
      "recordingSha256": "94231f19bdef4178eaf377c024940516851a8bb80791511fc5b5dff62f804d87",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-10-30T04-41-50-457Z-1glo/metrics.json",
      "metricsSha256": "ccf91f884f8daaf112a06c307caf48f1b865be44218cf4bf4e9d1e117a2dcd46",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 893,
        "sampleCount": 30,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 30,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T04:42:12.547Z",
            "sessionId": "2532955f-5f8c-41d7-b17e-bf83fe07b81c",
            "source": "first-token-detail",
            "tokensPerSec": 1080,
            "ttftMs": 15616
          },
          {
            "line": 189,
            "rx": "2026-10-30T04:42:53.078Z",
            "sessionId": "0a6b4702-0a79-45fc-bd46-683ea80d0896",
            "source": "first-token-detail",
            "tokensPerSec": 1312,
            "ttftMs": 6107
          },
          {
            "line": 349,
            "rx": "2026-10-30T04:43:34.119Z",
            "sessionId": "9402c5b9-b6ef-4185-bc7d-8a0ee629762e",
            "source": "first-token-detail",
            "tokensPerSec": 1289,
            "ttftMs": 7145
          },
          {
            "line": 472,
            "rx": "2026-10-30T04:44:03.891Z",
            "sessionId": "a9edbf3c-84cf-4a43-be5d-04e965ac26e3",
            "source": "first-token-detail",
            "tokensPerSec": 1203,
            "ttftMs": 6934
          },
          {
            "line": 657,
            "rx": "2026-10-30T04:44:55.026Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 1117,
            "ttftMs": 12500
          },
          {
            "line": 2341,
            "rx": "2026-10-30T04:50:56.584Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 407,
            "ttftMs": 405
          },
          {
            "line": 2987,
            "rx": "2026-10-30T04:52:43.411Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 462,
            "ttftMs": 635
          },
          {
            "line": 4085,
            "rx": "2026-10-30T04:56:10.402Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 953,
            "ttftMs": 23717
          },
          {
            "line": 4745,
            "rx": "2026-10-30T05:00:14.686Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 406,
            "ttftMs": 758
          },
          {
            "line": 4976,
            "rx": "2026-10-30T05:00:53.716Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 691,
            "ttftMs": 601
          },
          {
            "line": 5997,
            "rx": "2026-10-30T05:04:11.181Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 933,
            "ttftMs": 32124
          },
          {
            "line": 6097,
            "rx": "2026-10-30T05:05:08.447Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 879,
            "ttftMs": 39734
          },
          {
            "line": 6437,
            "rx": "2026-10-30T05:06:02.741Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 441,
            "ttftMs": 789
          },
          {
            "line": 6817,
            "rx": "2026-10-30T05:07:02.725Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 379,
            "ttftMs": 619
          },
          {
            "line": 7206,
            "rx": "2026-10-30T05:08:05.106Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 367,
            "ttftMs": 621
          },
          {
            "line": 7518,
            "rx": "2026-10-30T05:09:28.886Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 957,
            "ttftMs": 32969
          },
          {
            "line": 7693,
            "rx": "2026-10-30T05:11:52.511Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 1297,
            "ttftMs": 25544
          },
          {
            "line": 7823,
            "rx": "2026-10-30T05:13:08.820Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 1142,
            "ttftMs": 34634
          },
          {
            "line": 8004,
            "rx": "2026-10-30T05:14:31.041Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 1055,
            "ttftMs": 33138
          },
          {
            "line": 8163,
            "rx": "2026-10-30T05:16:12.958Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 886,
            "ttftMs": 46715
          },
          {
            "line": 8396,
            "rx": "2026-10-30T05:19:50.033Z",
            "sessionId": "e8924ca9-635e-4c81-9feb-3833dd9f81d6",
            "source": "first-token-detail",
            "tokensPerSec": 1816,
            "ttftMs": 7273
          },
          {
            "line": 9650,
            "rx": "2026-10-30T05:23:17.303Z",
            "sessionId": "e8924ca9-635e-4c81-9feb-3833dd9f81d6",
            "source": "first-token-detail",
            "tokensPerSec": 328,
            "ttftMs": 470
          },
          {
            "line": 9735,
            "rx": "2026-10-30T05:23:38.234Z",
            "sessionId": "e5272d5e-53af-41a8-8d59-aee29bce8df4",
            "source": "first-token-detail",
            "tokensPerSec": 900,
            "ttftMs": 20589
          },
          {
            "line": 9880,
            "rx": "2026-10-30T05:26:55.038Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 750,
            "ttftMs": 589
          },
          {
            "line": 10061,
            "rx": "2026-10-30T05:27:48.904Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 1246,
            "ttftMs": 29980
          },
          {
            "line": 10182,
            "rx": "2026-10-30T05:29:20.844Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 1069,
            "ttftMs": 40981
          },
          {
            "line": 10344,
            "rx": "2026-10-30T05:30:36.592Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 870,
            "ttftMs": 51877
          },
          {
            "line": 10660,
            "rx": "2026-10-30T05:31:34.506Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 316,
            "ttftMs": 935
          },
          {
            "line": 10877,
            "rx": "2026-10-30T05:34:50.580Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 629,
            "ttftMs": 5469
          },
          {
            "line": 11349,
            "rx": "2026-10-30T05:36:06.539Z",
            "sessionId": "3eb74780-06b5-4a56-bf5c-a5855cb6f723",
            "source": "first-token-detail",
            "tokensPerSec": 407,
            "ttftMs": 950
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-wikipedia-research-brief",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "wikipedia-research-brief",
        "repetition": 2
      },
      "trialId": "wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-30T05-37-22-994Z-2803",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-30T05-37-22-994Z-2803/recording/chat-events.jsonl",
      "recordingSha256": "f76d9d515a6efaee12a234e7e5675fde1e2b18410140f86153854e55a4064983",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-10-30T05-37-22-994Z-2803/metrics.json",
      "metricsSha256": "85167934c1ec7a02524ef18427d17970dc5af35b6d994765823fdd666dfdb2fa",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 530,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-10-30T05:37:36.035Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 11340
          },
          {
            "line": 512,
            "rx": "2026-10-30T05:39:07.695Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 620,
            "ttftMs": 8211
          },
          {
            "line": 1054,
            "rx": "2026-10-30T05:40:53.104Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 561,
            "ttftMs": 8284
          },
          {
            "line": 1209,
            "rx": "2026-10-30T05:41:42.474Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 530,
            "ttftMs": 12508
          },
          {
            "line": 1279,
            "rx": "2026-10-30T05:42:26.080Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 15979
          },
          {
            "line": 1337,
            "rx": "2026-10-30T05:43:15.699Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 517,
            "ttftMs": 20436
          },
          {
            "line": 1399,
            "rx": "2026-10-30T05:44:14.764Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 24309
          },
          {
            "line": 1486,
            "rx": "2026-10-30T05:45:25.771Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 487,
            "ttftMs": 30096
          },
          {
            "line": 1585,
            "rx": "2026-10-30T05:46:45.586Z",
            "sessionId": "af8e922c-4179-4097-95de-b1cf15b8885d",
            "source": "first-token-detail",
            "tokensPerSec": 486,
            "ttftMs": 34664
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-wikipedia-research-brief",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "wikipedia-research-brief",
        "repetition": 2
      },
      "trialId": "wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-30T05-54-48-778Z-fskp",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-30T05-54-48-778Z-fskp/recording/chat-events.jsonl",
      "recordingSha256": "769b0a8b291b01a06e9e4facbd29187d73f0ad2de713f8682bc176ad32ddb8d9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-gemma4-12b-q4-2026-10-30T05-54-48-778Z-fskp/metrics.json",
      "metricsSha256": "bd9e95e92eae5fb37ae7eb990df62dec30e15a6d4cd2c7844b6dae6e8812828a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1068,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T05:55:01.369Z",
            "sessionId": "4727b30a-86a4-42b5-9ad6-9fe7c14c9eee",
            "source": "first-token-detail",
            "tokensPerSec": 1068,
            "ttftMs": 10871
          },
          {
            "line": 176,
            "rx": "2026-10-30T05:55:55.656Z",
            "sessionId": "4727b30a-86a4-42b5-9ad6-9fe7c14c9eee",
            "source": "first-token-detail",
            "tokensPerSec": 1385,
            "ttftMs": 5394
          },
          {
            "line": 268,
            "rx": "2026-10-30T05:56:36.795Z",
            "sessionId": "4727b30a-86a4-42b5-9ad6-9fe7c14c9eee",
            "source": "first-token-detail",
            "tokensPerSec": 1314,
            "ttftMs": 6343
          },
          {
            "line": 356,
            "rx": "2026-10-30T05:57:02.134Z",
            "sessionId": "4727b30a-86a4-42b5-9ad6-9fe7c14c9eee",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 1539
          },
          {
            "line": 438,
            "rx": "2026-10-30T05:57:27.213Z",
            "sessionId": "4727b30a-86a4-42b5-9ad6-9fe7c14c9eee",
            "source": "first-token-detail",
            "tokensPerSec": 1022,
            "ttftMs": 1502
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-craftbook-week-plan",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-week-plan",
        "repetition": 2
      },
      "trialId": "craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-30T07-33-57-222Z-0akv",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-30T07-33-57-222Z-0akv/recording/chat-events.jsonl",
      "recordingSha256": "3ff46f591e30976c7b7d1172055656744fefeb859a2b5dd7437cec15ff968250",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-gemma4-12b-q4-2026-10-30T07-33-57-222Z-0akv/metrics.json",
      "metricsSha256": "7ca650c62bb33e7138b7e5fa74fb35e8b73b1132a42f72a00c78148937dbc9c2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1190,
        "sampleCount": 37,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 37,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T07:34:21.522Z",
            "sessionId": "dc6c2c52-823b-45ce-b6eb-d5b64b61fb2c",
            "source": "first-token-detail",
            "tokensPerSec": 1089,
            "ttftMs": 17748
          },
          {
            "line": 168,
            "rx": "2026-10-30T07:34:59.169Z",
            "sessionId": "dc6c2c52-823b-45ce-b6eb-d5b64b61fb2c",
            "source": "first-token-detail",
            "tokensPerSec": 1218,
            "ttftMs": 10378
          },
          {
            "line": 767,
            "rx": "2026-10-30T07:36:33.979Z",
            "sessionId": "dc6c2c52-823b-45ce-b6eb-d5b64b61fb2c",
            "source": "first-token-detail",
            "tokensPerSec": 665,
            "ttftMs": 695
          },
          {
            "line": 1130,
            "rx": "2026-10-30T07:37:40.788Z",
            "sessionId": "c0f0649c-bb2c-4486-bd79-c27661f90e88",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 6920
          },
          {
            "line": 1299,
            "rx": "2026-10-30T07:38:21.573Z",
            "sessionId": "7d5a2403-54f4-498f-979c-f3dd1ae0b015",
            "source": "first-token-detail",
            "tokensPerSec": 1149,
            "ttftMs": 7715
          },
          {
            "line": 1462,
            "rx": "2026-10-30T07:39:04.713Z",
            "sessionId": "85047cf2-f9a4-4412-a595-110609420517",
            "source": "first-token-detail",
            "tokensPerSec": 1250,
            "ttftMs": 5871
          },
          {
            "line": 1744,
            "rx": "2026-10-30T07:40:04.357Z",
            "sessionId": "85047cf2-f9a4-4412-a595-110609420517",
            "source": "first-token-detail",
            "tokensPerSec": 595,
            "ttftMs": 1073
          },
          {
            "line": 2070,
            "rx": "2026-10-30T07:41:20.589Z",
            "sessionId": "a2cba394-3ea2-47c4-984e-1ddaf930317d",
            "source": "first-token-detail",
            "tokensPerSec": 1147,
            "ttftMs": 6673
          },
          {
            "line": 2239,
            "rx": "2026-10-30T07:41:59.775Z",
            "sessionId": "c8511eed-472f-4fa8-99e6-163110139305",
            "source": "first-token-detail",
            "tokensPerSec": 1295,
            "ttftMs": 5841
          },
          {
            "line": 2454,
            "rx": "2026-10-30T07:42:50.542Z",
            "sessionId": "b27ec46c-017c-4bc8-a786-1276bc7206aa",
            "source": "first-token-detail",
            "tokensPerSec": 1190,
            "ttftMs": 6603
          },
          {
            "line": 2777,
            "rx": "2026-10-30T07:43:54.685Z",
            "sessionId": "b27ec46c-017c-4bc8-a786-1276bc7206aa",
            "source": "first-token-detail",
            "tokensPerSec": 1031,
            "ttftMs": 10342
          },
          {
            "line": 3332,
            "rx": "2026-10-30T07:45:23.110Z",
            "sessionId": "b27ec46c-017c-4bc8-a786-1276bc7206aa",
            "source": "first-token-detail",
            "tokensPerSec": 762,
            "ttftMs": 792
          },
          {
            "line": 3797,
            "rx": "2026-10-30T07:46:50.233Z",
            "sessionId": "19af2b18-cc3f-4638-b49b-662457c1714b",
            "source": "first-token-detail",
            "tokensPerSec": 1310,
            "ttftMs": 6125
          },
          {
            "line": 4091,
            "rx": "2026-10-30T07:47:55.763Z",
            "sessionId": "19af2b18-cc3f-4638-b49b-662457c1714b",
            "source": "first-token-detail",
            "tokensPerSec": 592,
            "ttftMs": 1185
          },
          {
            "line": 4453,
            "rx": "2026-10-30T07:49:10.976Z",
            "sessionId": "cff52633-82d3-491b-a2c9-f86d15b92f26",
            "source": "first-token-detail",
            "tokensPerSec": 1241,
            "ttftMs": 6833
          },
          {
            "line": 6291,
            "rx": "2026-10-30T07:54:28.409Z",
            "sessionId": "f8a6737b-4f76-4c94-a8a7-994d6dae687a",
            "source": "first-token-detail",
            "tokensPerSec": 1079,
            "ttftMs": 9260
          },
          {
            "line": 8017,
            "rx": "2026-10-30T07:59:04.916Z",
            "sessionId": "f8a6737b-4f76-4c94-a8a7-994d6dae687a",
            "source": "first-token-detail",
            "tokensPerSec": 288,
            "ttftMs": 2674
          },
          {
            "line": 8137,
            "rx": "2026-10-30T07:59:21.845Z",
            "sessionId": "ad98c35a-a7be-4056-add8-1da193e81ead",
            "source": "first-token-detail",
            "tokensPerSec": 32,
            "ttftMs": 302682
          },
          {
            "line": 8598,
            "rx": "2026-10-30T08:01:06.232Z",
            "sessionId": "77fc6022-1887-487f-a161-f332d1012ae4",
            "source": "first-token-detail",
            "tokensPerSec": 1274,
            "ttftMs": 6910
          },
          {
            "line": 8842,
            "rx": "2026-10-30T08:02:49.519Z",
            "sessionId": "7aa0928d-a99d-4dac-bb48-d7e73e934c7e",
            "source": "first-token-detail",
            "tokensPerSec": 1686,
            "ttftMs": 5139
          },
          {
            "line": 10398,
            "rx": "2026-10-30T08:08:07.377Z",
            "sessionId": "5e8ea8c4-009a-4f17-919d-37b93a6f5739",
            "source": "first-token-detail",
            "tokensPerSec": 1146,
            "ttftMs": 7846
          },
          {
            "line": 12157,
            "rx": "2026-10-30T08:13:56.821Z",
            "sessionId": "e2aaa1b2-2d77-452d-9cdd-7aacd98e141b",
            "source": "first-token-detail",
            "tokensPerSec": 1217,
            "ttftMs": 7181
          },
          {
            "line": 12343,
            "rx": "2026-10-30T08:15:34.578Z",
            "sessionId": "f758a59c-fcc5-439d-b72f-1143aaa01f8a",
            "source": "first-token-detail",
            "tokensPerSec": 1860,
            "ttftMs": 4907
          },
          {
            "line": 12775,
            "rx": "2026-10-30T08:17:04.796Z",
            "sessionId": "b4e4dfcb-a8ab-466e-a432-b2eb5a03011f",
            "source": "first-token-detail",
            "tokensPerSec": 1775,
            "ttftMs": 5067
          },
          {
            "line": 12997,
            "rx": "2026-10-30T08:18:56.827Z",
            "sessionId": "d57a1a3a-c975-4550-aa0a-e9032a700a26",
            "source": "first-token-detail",
            "tokensPerSec": 1319,
            "ttftMs": 7087
          },
          {
            "line": 13139,
            "rx": "2026-10-30T08:20:24.458Z",
            "sessionId": "9bf59495-bf1d-4813-adc8-7aeefbf08e8c",
            "source": "first-token-detail",
            "tokensPerSec": 1887,
            "ttftMs": 4780
          },
          {
            "line": 14797,
            "rx": "2026-10-30T08:25:57.992Z",
            "sessionId": "15bec8d8-3259-44c4-9989-9cb434cb7229",
            "source": "first-token-detail",
            "tokensPerSec": 1147,
            "ttftMs": 8148
          },
          {
            "line": 14963,
            "rx": "2026-10-30T08:27:34.662Z",
            "sessionId": "614ea832-11b3-44d9-990a-41c004f003e8",
            "source": "first-token-detail",
            "tokensPerSec": 1868,
            "ttftMs": 4811
          },
          {
            "line": 15203,
            "rx": "2026-10-30T08:28:17.648Z",
            "sessionId": "614ea832-11b3-44d9-990a-41c004f003e8",
            "source": "first-token-detail",
            "tokensPerSec": 1012,
            "ttftMs": 664
          },
          {
            "line": 15567,
            "rx": "2026-10-30T08:30:31.715Z",
            "sessionId": "52538833-3fb4-4872-967d-8ebfe5e8a443",
            "source": "first-token-detail",
            "tokensPerSec": 1419,
            "ttftMs": 6713
          },
          {
            "line": 15659,
            "rx": "2026-10-30T08:31:54.876Z",
            "sessionId": "536b5673-7667-48be-a8a2-1babf71b7ed4",
            "source": "first-token-detail",
            "tokensPerSec": 1879,
            "ttftMs": 4873
          },
          {
            "line": 15776,
            "rx": "2026-10-30T08:33:21.538Z",
            "sessionId": "536b5673-7667-48be-a8a2-1babf71b7ed4",
            "source": "first-token-detail",
            "tokensPerSec": 1814,
            "ttftMs": 6493
          },
          {
            "line": 15956,
            "rx": "2026-10-30T08:34:55.413Z",
            "sessionId": "ad227b2e-9a38-4c48-8b68-594845329f66",
            "source": "first-token-detail",
            "tokensPerSec": 1840,
            "ttftMs": 5320
          },
          {
            "line": 16119,
            "rx": "2026-10-30T08:35:27.183Z",
            "sessionId": "ad227b2e-9a38-4c48-8b68-594845329f66",
            "source": "first-token-detail",
            "tokensPerSec": 1713,
            "ttftMs": 7416
          },
          {
            "line": 16718,
            "rx": "2026-10-30T08:37:51.844Z",
            "sessionId": "ad227b2e-9a38-4c48-8b68-594845329f66",
            "source": "first-token-detail",
            "tokensPerSec": 1124,
            "ttftMs": 1383
          },
          {
            "line": 17225,
            "rx": "2026-10-30T08:39:16.499Z",
            "sessionId": "ad227b2e-9a38-4c48-8b68-594845329f66",
            "source": "first-token-detail",
            "tokensPerSec": 548,
            "ttftMs": 466
          },
          {
            "line": 17892,
            "rx": "2026-10-30T08:41:07.713Z",
            "sessionId": "ad227b2e-9a38-4c48-8b68-594845329f66",
            "source": "first-token-detail",
            "tokensPerSec": 971,
            "ttftMs": 741
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-craftbook-week-plan",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-week-plan",
        "repetition": 2
      },
      "trialId": "craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-30T08-51-49-083Z-ebq2",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-30T08-51-49-083Z-ebq2/recording/chat-events.jsonl",
      "recordingSha256": "19e53cf1b7fbde4e6ae2c6e860c3adb16870784855551cd74e346b2bfe748318",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-10-30T08-51-49-083Z-ebq2/metrics.json",
      "metricsSha256": "425a23f927f0c9b17a861a0cfad11e1b8ab9689368cf1e8cf738a1f2cd763982",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 531,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 35,
            "rx": "2026-10-30T08:52:27.332Z",
            "sessionId": "b1c78d38-1f79-439d-8c72-2e50f46d4696",
            "source": "first-token-detail",
            "tokensPerSec": 588,
            "ttftMs": 31647
          },
          {
            "line": 5005,
            "rx": "2026-10-30T09:07:37.957Z",
            "sessionId": "b1c78d38-1f79-439d-8c72-2e50f46d4696",
            "source": "first-token-detail",
            "tokensPerSec": 467,
            "ttftMs": 28640
          },
          {
            "line": 5895,
            "rx": "2026-10-30T09:10:47.860Z",
            "sessionId": "b1c78d38-1f79-439d-8c72-2e50f46d4696",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 31752
          },
          {
            "line": 7495,
            "rx": "2026-10-30T09:15:37.658Z",
            "sessionId": "a3249fec-e017-49b9-b8ed-021200f29427",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 16351
          },
          {
            "line": 8292,
            "rx": "2026-10-30T09:18:26.900Z",
            "sessionId": "24d8a29e-d17b-48ed-91fb-f8639b049048",
            "source": "first-token-detail",
            "tokensPerSec": 556,
            "ttftMs": 15575
          },
          {
            "line": 10002,
            "rx": "2026-10-30T09:23:49.630Z",
            "sessionId": "e14b057b-b93a-42f5-90aa-ab55f2381c76",
            "source": "first-token-detail",
            "tokensPerSec": 593,
            "ttftMs": 8187
          },
          {
            "line": 10532,
            "rx": "2026-10-30T09:25:56.062Z",
            "sessionId": "bf249c13-d269-4b92-be05-d7bcc4f347f5",
            "source": "first-token-detail",
            "tokensPerSec": 527,
            "ttftMs": 9615
          },
          {
            "line": 12917,
            "rx": "2026-10-30T09:33:21.891Z",
            "sessionId": "20cee4a2-675b-467d-bcce-105a0055e6a1",
            "source": "first-token-detail",
            "tokensPerSec": 550,
            "ttftMs": 10245
          },
          {
            "line": 13182,
            "rx": "2026-10-30T09:34:52.668Z",
            "sessionId": "20cee4a2-675b-467d-bcce-105a0055e6a1",
            "source": "first-token-detail",
            "tokensPerSec": 518,
            "ttftMs": 33485
          },
          {
            "line": 14070,
            "rx": "2026-10-30T09:37:45.825Z",
            "sessionId": "20cee4a2-675b-467d-bcce-105a0055e6a1",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 1404
          },
          {
            "line": 14504,
            "rx": "2026-10-30T09:40:18.290Z",
            "sessionId": "20cee4a2-675b-467d-bcce-105a0055e6a1",
            "source": "first-token-detail",
            "tokensPerSec": 585,
            "ttftMs": 32675
          },
          {
            "line": 14836,
            "rx": "2026-10-30T09:41:49.848Z",
            "sessionId": "20cee4a2-675b-467d-bcce-105a0055e6a1",
            "source": "first-token-detail",
            "tokensPerSec": 563,
            "ttftMs": 38898
          },
          {
            "line": 15195,
            "rx": "2026-10-30T09:44:03.095Z",
            "sessionId": "053738a6-7c29-43c1-924a-0f09e651730a",
            "source": "first-token-detail",
            "tokensPerSec": 586,
            "ttftMs": 11261
          },
          {
            "line": 19376,
            "rx": "2026-10-30T09:57:06.655Z",
            "sessionId": "bd4712be-38df-4290-8ec6-9a6f3f845f1d",
            "source": "first-token-detail",
            "tokensPerSec": 760,
            "ttftMs": 9486
          },
          {
            "line": 19602,
            "rx": "2026-10-30T09:57:54.354Z",
            "sessionId": "bd4712be-38df-4290-8ec6-9a6f3f845f1d",
            "source": "first-token-detail",
            "tokensPerSec": 406,
            "ttftMs": 879
          },
          {
            "line": 19830,
            "rx": "2026-10-30T09:58:39.771Z",
            "sessionId": "bd4712be-38df-4290-8ec6-9a6f3f845f1d",
            "source": "first-token-detail",
            "tokensPerSec": 289,
            "ttftMs": 1133
          },
          {
            "line": 20217,
            "rx": "2026-10-30T10:01:03.034Z",
            "sessionId": "bd4712be-38df-4290-8ec6-9a6f3f845f1d",
            "source": "first-token-detail",
            "tokensPerSec": 429,
            "ttftMs": 76272
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-craftbook-ab-test-readout",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-ab-test-readout",
        "repetition": 2
      },
      "trialId": "craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-30T10-02-49-828Z-y69i",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-30T10-02-49-828Z-y69i/recording/chat-events.jsonl",
      "recordingSha256": "970f6077bbe3d309b26fce2dd3be88ba48e656e50a28056bdcd3ac273b262fe1",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-10-30T10-02-49-828Z-y69i/metrics.json",
      "metricsSha256": "f4ff485b9c1d51f8d26ac756c73b60e65f60b4a60a090d6db8853856f3c44989",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1096.5,
        "sampleCount": 28,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 28,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T10:03:13.675Z",
            "sessionId": "e4ba8c75-16b7-4cea-9303-0a0bd3d6a74f",
            "source": "first-token-detail",
            "tokensPerSec": 1030,
            "ttftMs": 17336
          },
          {
            "line": 195,
            "rx": "2026-10-30T10:03:54.626Z",
            "sessionId": "e4ba8c75-16b7-4cea-9303-0a0bd3d6a74f",
            "source": "first-token-detail",
            "tokensPerSec": 1085,
            "ttftMs": 13291
          },
          {
            "line": 587,
            "rx": "2026-10-30T10:05:08.211Z",
            "sessionId": "e03b169b-83d4-41a3-8571-c9c919bbdf15",
            "source": "first-token-detail",
            "tokensPerSec": 1273,
            "ttftMs": 6764
          },
          {
            "line": 697,
            "rx": "2026-10-30T10:05:28.812Z",
            "sessionId": "e03b169b-83d4-41a3-8571-c9c919bbdf15",
            "source": "first-token-detail",
            "tokensPerSec": 587,
            "ttftMs": 852
          },
          {
            "line": 994,
            "rx": "2026-10-30T10:06:40.219Z",
            "sessionId": "67cc9de5-727d-41d9-85cd-63c3e412b53b",
            "source": "first-token-detail",
            "tokensPerSec": 1078,
            "ttftMs": 8740
          },
          {
            "line": 1126,
            "rx": "2026-10-30T10:07:12.880Z",
            "sessionId": "da87cfda-a3c1-44ca-8a43-21ced05a9fb2",
            "source": "first-token-detail",
            "tokensPerSec": 1214,
            "ttftMs": 6484
          },
          {
            "line": 1328,
            "rx": "2026-10-30T10:08:08.348Z",
            "sessionId": "a08a718c-a4c1-4192-adc8-34c69281bac0",
            "source": "first-token-detail",
            "tokensPerSec": 1185,
            "ttftMs": 6918
          },
          {
            "line": 1468,
            "rx": "2026-10-30T10:08:43.729Z",
            "sessionId": "65d1b923-4799-4e3b-9bb3-ae583e99fd26",
            "source": "first-token-detail",
            "tokensPerSec": 1100,
            "ttftMs": 7342
          },
          {
            "line": 3243,
            "rx": "2026-10-30T10:13:43.749Z",
            "sessionId": "b579e6a4-ba8b-4625-ba63-abffd086449b",
            "source": "first-token-detail",
            "tokensPerSec": 1173,
            "ttftMs": 7126
          },
          {
            "line": 3379,
            "rx": "2026-10-30T10:14:18.142Z",
            "sessionId": "10e178bd-4199-4001-9d7c-8b32f5640e01",
            "source": "first-token-detail",
            "tokensPerSec": 1272,
            "ttftMs": 6548
          },
          {
            "line": 3488,
            "rx": "2026-10-30T10:14:39.924Z",
            "sessionId": "10e178bd-4199-4001-9d7c-8b32f5640e01",
            "source": "first-token-detail",
            "tokensPerSec": 617,
            "ttftMs": 948
          },
          {
            "line": 3666,
            "rx": "2026-10-30T10:15:35.200Z",
            "sessionId": "db815fa1-929b-471d-ab92-0e1337d4da06",
            "source": "first-token-detail",
            "tokensPerSec": 1022,
            "ttftMs": 8566
          },
          {
            "line": 4378,
            "rx": "2026-10-30T10:18:09.028Z",
            "sessionId": "865ccd8e-934d-4558-baad-5dfb182c3392",
            "source": "first-token-detail",
            "tokensPerSec": 1182,
            "ttftMs": 7306
          },
          {
            "line": 4559,
            "rx": "2026-10-30T10:18:41.630Z",
            "sessionId": "865ccd8e-934d-4558-baad-5dfb182c3392",
            "source": "first-token-detail",
            "tokensPerSec": 643,
            "ttftMs": 864
          },
          {
            "line": 4916,
            "rx": "2026-10-30T10:20:04.986Z",
            "sessionId": "ea3a7357-b74f-486a-b591-6aaee1cdb9c4",
            "source": "first-token-detail",
            "tokensPerSec": 1102,
            "ttftMs": 8228
          },
          {
            "line": 5881,
            "rx": "2026-10-30T10:23:05.669Z",
            "sessionId": "b7487289-313f-4fd9-b354-7e1001a9677c",
            "source": "first-token-detail",
            "tokensPerSec": 1093,
            "ttftMs": 9089
          },
          {
            "line": 6228,
            "rx": "2026-10-30T10:24:04.327Z",
            "sessionId": "b7487289-313f-4fd9-b354-7e1001a9677c",
            "source": "first-token-detail",
            "tokensPerSec": 362,
            "ttftMs": 1071
          },
          {
            "line": 6560,
            "rx": "2026-10-30T10:24:52.360Z",
            "sessionId": "729eba2f-0c94-4d51-97fb-281c65f74141",
            "source": "first-token-detail",
            "tokensPerSec": 135,
            "ttftMs": 115620
          },
          {
            "line": 6585,
            "rx": "2026-10-30T10:24:54.123Z",
            "sessionId": "b7487289-313f-4fd9-b354-7e1001a9677c",
            "source": "first-token-detail",
            "tokensPerSec": 351,
            "ttftMs": 456
          },
          {
            "line": 7046,
            "rx": "2026-10-30T10:26:19.538Z",
            "sessionId": "81833729-c9d0-4cb4-862e-3bf7a09aaff9",
            "source": "first-token-detail",
            "tokensPerSec": 1213,
            "ttftMs": 7636
          },
          {
            "line": 7226,
            "rx": "2026-10-30T10:27:06.308Z",
            "sessionId": "81833729-c9d0-4cb4-862e-3bf7a09aaff9",
            "source": "first-token-detail",
            "tokensPerSec": 1181,
            "ttftMs": 9757
          },
          {
            "line": 8093,
            "rx": "2026-10-30T10:29:44.470Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 1254,
            "ttftMs": 7508
          },
          {
            "line": 8233,
            "rx": "2026-10-30T10:30:23.394Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 1188,
            "ttftMs": 9695
          },
          {
            "line": 9925,
            "rx": "2026-10-30T10:35:04.218Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 1055,
            "ttftMs": 11915
          },
          {
            "line": 10585,
            "rx": "2026-10-30T10:36:57.239Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 597,
            "ttftMs": 1200
          },
          {
            "line": 11166,
            "rx": "2026-10-30T10:38:44.444Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 1106,
            "ttftMs": 13020
          },
          {
            "line": 11824,
            "rx": "2026-10-30T10:40:37.118Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 490
          },
          {
            "line": 12155,
            "rx": "2026-10-30T10:41:38.173Z",
            "sessionId": "d5c356bd-a1d4-4cbf-817a-dbc6ff6347b6",
            "source": "first-token-detail",
            "tokensPerSec": 1153,
            "ttftMs": 1173
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-craftbook-ab-test-readout",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-ab-test-readout",
        "repetition": 2
      },
      "trialId": "craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-30T11-24-14-783Z-5et1",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-30T11-24-14-783Z-5et1/recording/chat-events.jsonl",
      "recordingSha256": "74a1e26656453f0db6a20cf01bbf3485e804af17eaca0c4d44ad52237b0ba2cf",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-10-30T11-24-14-783Z-5et1/metrics.json",
      "metricsSha256": "f1e4060d7412d26051da474f7bda2c414014b48fd577671f7e451e475d08dff4",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 577.5,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 35,
            "rx": "2026-10-30T11:24:51.035Z",
            "sessionId": "b05d8994-dd0f-45c8-beaa-a58d5be552ec",
            "source": "first-token-detail",
            "tokensPerSec": 554,
            "ttftMs": 29741
          },
          {
            "line": 561,
            "rx": "2026-10-30T11:26:43.362Z",
            "sessionId": "b05d8994-dd0f-45c8-beaa-a58d5be552ec",
            "source": "first-token-detail",
            "tokensPerSec": 586,
            "ttftMs": 26932
          },
          {
            "line": 7123,
            "rx": "2026-10-30T11:46:01.655Z",
            "sessionId": "782ed65e-0cbc-4993-83e2-063909e3082b",
            "source": "first-token-detail",
            "tokensPerSec": 534,
            "ttftMs": 14965
          },
          {
            "line": 7769,
            "rx": "2026-10-30T11:49:02.156Z",
            "sessionId": "7f1c1813-3b54-40b1-94ba-3eff72f0a6d6",
            "source": "first-token-detail",
            "tokensPerSec": 854,
            "ttftMs": 10276
          },
          {
            "line": 10052,
            "rx": "2026-10-30T11:56:25.751Z",
            "sessionId": "39a6c90f-f828-4348-96b7-79d0f94a36f3",
            "source": "first-token-detail",
            "tokensPerSec": 569,
            "ttftMs": 8703
          },
          {
            "line": 10777,
            "rx": "2026-10-30T11:59:31.216Z",
            "sessionId": "579ccb5a-35c2-4723-8965-2043e836b026",
            "source": "first-token-detail",
            "tokensPerSec": 628,
            "ttftMs": 9132
          },
          {
            "line": 15285,
            "rx": "2026-10-30T12:13:30.029Z",
            "sessionId": "b154930a-33f9-4377-b1a6-3cc50f5a5761",
            "source": "first-token-detail",
            "tokensPerSec": 786,
            "ttftMs": 12648
          },
          {
            "line": 17191,
            "rx": "2026-10-30T12:19:20.447Z",
            "sessionId": "8c211d4f-c3c9-4932-9ae9-b1406790fed0",
            "source": "first-token-detail",
            "tokensPerSec": 470,
            "ttftMs": 26711
          },
          {
            "line": 17787,
            "rx": "2026-10-30T12:21:11.760Z",
            "sessionId": "903330a5-d66d-4f8d-aaee-1bd86783551b",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 8238
          },
          {
            "line": 17839,
            "rx": "2026-10-30T12:21:32.177Z",
            "sessionId": "903330a5-d66d-4f8d-aaee-1bd86783551b",
            "source": "first-token-detail",
            "tokensPerSec": 658,
            "ttftMs": 3194
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-craftbook-annotated-bibliography",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-annotated-bibliography",
        "repetition": 2
      },
      "trialId": "craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-30T12-44-37-552Z-8w4x",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-30T12-44-37-552Z-8w4x/recording/chat-events.jsonl",
      "recordingSha256": "6a83f86be6d02bd4c0e086ac3b17c686741160d6115609d7229ceedca04a48a8",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-10-30T12-44-37-552Z-8w4x/metrics.json",
      "metricsSha256": "f29545a6f6b53f85b10ee875288c7e6183d86e516b64f5095f77051441573b8b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 560,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-30T12:45:04.196Z",
            "sessionId": "26404adc-54ef-47a3-ae69-1c0cad044cb3",
            "source": "first-token-detail",
            "tokensPerSec": 592,
            "ttftMs": 20123
          },
          {
            "line": 329,
            "rx": "2026-10-30T12:46:13.773Z",
            "sessionId": "f1985187-919e-4370-a4de-f602628fd710",
            "source": "first-token-detail",
            "tokensPerSec": 629,
            "ttftMs": 19586
          },
          {
            "line": 2555,
            "rx": "2026-10-30T12:53:04.836Z",
            "sessionId": "0feaa08c-8ec9-43cd-a776-8a3197e886ee",
            "source": "first-token-detail",
            "tokensPerSec": 528,
            "ttftMs": 15520
          },
          {
            "line": 2970,
            "rx": "2026-10-30T12:54:36.118Z",
            "sessionId": "7f573e65-fa74-4425-ab71-28bac9b63b5b",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 16900
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-craftbook-annotated-bibliography",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-annotated-bibliography",
        "repetition": 2
      },
      "trialId": "craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-30T13-03-04-848Z-d9qz",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-30T13-03-04-848Z-d9qz/recording/chat-events.jsonl",
      "recordingSha256": "adec8544977c889a4124088127530a9898e1478be37874ba7ebc6f3b7e1bf963",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-10-30T13-03-04-848Z-d9qz/metrics.json",
      "metricsSha256": "d7d89dc647c0773494318210540cd7cdb77528cc1b9fa1683b8c2b692f4c02ad",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1238,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T13:03:25.095Z",
            "sessionId": "17be585b-6c0a-4901-ac09-59817be918a3",
            "source": "first-token-detail",
            "tokensPerSec": 1103,
            "ttftMs": 13747
          },
          {
            "line": 166,
            "rx": "2026-10-30T13:04:00.193Z",
            "sessionId": "9b311d79-713e-4257-9ced-722e75412d78",
            "source": "first-token-detail",
            "tokensPerSec": 1225,
            "ttftMs": 8813
          },
          {
            "line": 401,
            "rx": "2026-10-30T13:05:08.281Z",
            "sessionId": "cb081b3f-ef45-4752-8cca-47f3279ead37",
            "source": "first-token-detail",
            "tokensPerSec": 1287,
            "ttftMs": 6868
          },
          {
            "line": 563,
            "rx": "2026-10-30T13:05:53.203Z",
            "sessionId": "933df57b-fed3-4951-afac-d7b82ccc4f68",
            "source": "first-token-detail",
            "tokensPerSec": 1251,
            "ttftMs": 6866
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-productivity-craftbook-research-to-document",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-research-to-document",
        "repetition": 2
      },
      "trialId": "craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-30T13-06-37-068Z-br3m",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-30T13-06-37-068Z-br3m/recording/chat-events.jsonl",
      "recordingSha256": "4e8da2fd7fe1cef63460bb661d6f1b4a8ff78afe401766458b349186dd73a7db",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-10-30T13-06-37-068Z-br3m/metrics.json",
      "metricsSha256": "6c1160862abdbe720772630f334a8971948b18d61ed6d9ad7829e36a8833a9c9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 588,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-30T13:07:03.603Z",
            "sessionId": "d81d5a5a-c07a-4b54-8765-294a6f20a4d3",
            "source": "first-token-detail",
            "tokensPerSec": 588,
            "ttftMs": 20024
          },
          {
            "line": 208,
            "rx": "2026-10-30T13:07:47.885Z",
            "sessionId": "728c3112-d884-4548-a61d-27cbe5a5eabf",
            "source": "first-token-detail",
            "tokensPerSec": 642,
            "ttftMs": 14239
          },
          {
            "line": 435,
            "rx": "2026-10-30T13:08:42.447Z",
            "sessionId": "25272d01-dd32-4264-926c-e1f55bc81162",
            "source": "first-token-detail",
            "tokensPerSec": 615,
            "ttftMs": 13778
          },
          {
            "line": 654,
            "rx": "2026-10-30T13:10:11.581Z",
            "sessionId": "25272d01-dd32-4264-926c-e1f55bc81162",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 52380
          },
          {
            "line": 1139,
            "rx": "2026-10-30T13:16:22.079Z",
            "sessionId": "428ece51-3e4f-45c1-a733-d3d58b17e778",
            "source": "first-token-detail",
            "tokensPerSec": 487,
            "ttftMs": 52638
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-productivity-craftbook-research-to-document",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-research-to-document",
        "repetition": 2
      },
      "trialId": "craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-30T13-34-05-240Z-ibbe",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-30T13-34-05-240Z-ibbe/recording/chat-events.jsonl",
      "recordingSha256": "705f0e3610ab03e98317b7d1e7de7dd006c7c5cd7978c9bcac4432e52777d138",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-gemma4-12b-q4-2026-10-30T13-34-05-240Z-ibbe/metrics.json",
      "metricsSha256": "546db886b1dbbf9bdb5edbf731e85ab68edd6674aea6949f8dda9f4240b9056a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1171,
        "sampleCount": 14,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 14,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T13:34:25.264Z",
            "sessionId": "0f36ecba-6a0c-4806-848d-e7b17a2b6244",
            "source": "first-token-detail",
            "tokensPerSec": 1137,
            "ttftMs": 13500
          },
          {
            "line": 101,
            "rx": "2026-10-30T13:34:48.034Z",
            "sessionId": "2d91b23e-3085-471c-8433-e54b4f2beeed",
            "source": "first-token-detail",
            "tokensPerSec": 1448,
            "ttftMs": 6291
          },
          {
            "line": 213,
            "rx": "2026-10-30T13:35:12.817Z",
            "sessionId": "3755e422-b5c2-43d7-81a5-0ff341f6a06c",
            "source": "first-token-detail",
            "tokensPerSec": 1520,
            "ttftMs": 6041
          },
          {
            "line": 1641,
            "rx": "2026-10-30T13:39:01.794Z",
            "sessionId": "3755e422-b5c2-43d7-81a5-0ff341f6a06c",
            "source": "first-token-detail",
            "tokensPerSec": 376,
            "ttftMs": 420
          },
          {
            "line": 1922,
            "rx": "2026-10-30T13:40:16.598Z",
            "sessionId": "3755e422-b5c2-43d7-81a5-0ff341f6a06c",
            "source": "first-token-detail",
            "tokensPerSec": 1196,
            "ttftMs": 14079
          },
          {
            "line": 2735,
            "rx": "2026-10-30T13:44:14.063Z",
            "sessionId": "3755e422-b5c2-43d7-81a5-0ff341f6a06c",
            "source": "first-token-detail",
            "tokensPerSec": 1146,
            "ttftMs": 19589
          },
          {
            "line": 3065,
            "rx": "2026-10-30T13:45:18.591Z",
            "sessionId": "b046cbf6-1b27-4e97-80eb-d7a05a0335a8",
            "source": "first-token-detail",
            "tokensPerSec": 1203,
            "ttftMs": 11175
          },
          {
            "line": 3275,
            "rx": "2026-10-30T13:46:59.543Z",
            "sessionId": "ec695ee4-8173-4925-a0d3-09420271ade5",
            "source": "first-token-detail",
            "tokensPerSec": 1815,
            "ttftMs": 7021
          },
          {
            "line": 3687,
            "rx": "2026-10-30T13:49:50.691Z",
            "sessionId": "712e2e11-62e8-42c1-9a81-f0806c344856",
            "source": "first-token-detail",
            "tokensPerSec": 1513,
            "ttftMs": 7988
          },
          {
            "line": 3906,
            "rx": "2026-10-30T13:51:29.850Z",
            "sessionId": "6d8bd994-026d-455c-af65-cc494894aaff",
            "source": "first-token-detail",
            "tokensPerSec": 1801,
            "ttftMs": 7134
          },
          {
            "line": 5916,
            "rx": "2026-10-30T13:57:13.461Z",
            "sessionId": "6d8bd994-026d-455c-af65-cc494894aaff",
            "source": "first-token-detail",
            "tokensPerSec": 422,
            "ttftMs": 647
          },
          {
            "line": 6266,
            "rx": "2026-10-30T13:58:22.145Z",
            "sessionId": "6d8bd994-026d-455c-af65-cc494894aaff",
            "source": "first-token-detail",
            "tokensPerSec": 1036,
            "ttftMs": 17799
          },
          {
            "line": 6663,
            "rx": "2026-10-30T13:59:48.566Z",
            "sessionId": "6d8bd994-026d-455c-af65-cc494894aaff",
            "source": "first-token-detail",
            "tokensPerSec": 980,
            "ttftMs": 21289
          },
          {
            "line": 8796,
            "rx": "2026-10-30T14:05:34.538Z",
            "sessionId": "5a8c33a2-4753-48a2-bc01-f9e63b5a5175",
            "source": "first-token-detail",
            "tokensPerSec": 1104,
            "ttftMs": 11552
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-fix-squisq-bugs",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "fix-squisq-bugs",
        "repetition": 2
      },
      "trialId": "fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-30T14-27-12-050Z-yaub",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-30T14-27-12-050Z-yaub/recording/chat-events.jsonl",
      "recordingSha256": "f9b229f65b5fa6932b0bc4297053379718b2d52e4e21ae989e30787d19c1a5f6",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-gemma4-12b-q4-2026-10-30T14-27-12-050Z-yaub/metrics.json",
      "metricsSha256": "8cdeaf6af0f8219d51370273f81ba1e74d19fafa321c4e445ae666e527d13ab2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1090,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T14:27:24.393Z",
            "sessionId": "b8a70d52-71f5-4551-96ce-9778fb3c03ea",
            "source": "first-token-detail",
            "tokensPerSec": 1090,
            "ttftMs": 10698
          },
          {
            "line": 453,
            "rx": "2026-10-30T14:29:13.937Z",
            "sessionId": "b8a70d52-71f5-4551-96ce-9778fb3c03ea",
            "source": "first-token-detail",
            "tokensPerSec": 1062,
            "ttftMs": 10578
          },
          {
            "line": 550,
            "rx": "2026-10-30T14:30:05.591Z",
            "sessionId": "b8a70d52-71f5-4551-96ce-9778fb3c03ea",
            "source": "first-token-detail",
            "tokensPerSec": 1203,
            "ttftMs": 11519
          },
          {
            "line": 601,
            "rx": "2026-10-30T14:30:20.493Z",
            "sessionId": "b8a70d52-71f5-4551-96ce-9778fb3c03ea",
            "source": "first-token-detail",
            "tokensPerSec": 978,
            "ttftMs": 1251
          },
          {
            "line": 657,
            "rx": "2026-10-30T14:31:03.825Z",
            "sessionId": "b8a70d52-71f5-4551-96ce-9778fb3c03ea",
            "source": "first-token-detail",
            "tokensPerSec": 1201,
            "ttftMs": 14409
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-fix-squisq-bugs",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "fix-squisq-bugs",
        "repetition": 2
      },
      "trialId": "fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-30T14-36-19-319Z-hj68",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-30T14-36-19-319Z-hj68/recording/chat-events.jsonl",
      "recordingSha256": "4ff2d8c7ef94e00ad37898b6d18dbd83a31d4e54b65249fc3dff5d6120e16a7c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-10-30T14-36-19-319Z-hj68/metrics.json",
      "metricsSha256": "52c67b927929c091c7e0443ffc98029fbac34d1a5c6cc195db4765640e7ddf38",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 567,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T14:36:34.375Z",
            "sessionId": "f05c0e13-b7f2-4efe-8fe1-ffb6ae51fc29",
            "source": "first-token-detail",
            "tokensPerSec": 567,
            "ttftMs": 13401
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-dev-craftbook-routing",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "dev-craftbook-routing",
        "repetition": 2
      },
      "trialId": "dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-30T14-38-23-678Z-zz8i",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-30T14-38-23-678Z-zz8i/recording/chat-events.jsonl",
      "recordingSha256": "95a4c6c37d2321206d95410f14c45cefbb9cd130e8f8010bc6d9ddcad6994a97",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-gemma4-12b-q4-2026-10-30T14-38-23-678Z-zz8i/metrics.json",
      "metricsSha256": "4696bf0569b7a1436e858a1d7f871f40c834277c99ff22fca52b68a277b67924",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 850,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T14:38:39.622Z",
            "sessionId": "60dd9e0b-6913-42b0-b9d4-c58c551e5028",
            "source": "first-token-detail",
            "tokensPerSec": 1270,
            "ttftMs": 14236
          },
          {
            "line": 1366,
            "rx": "2026-10-30T14:42:19.435Z",
            "sessionId": "60dd9e0b-6913-42b0-b9d4-c58c551e5028",
            "source": "first-token-detail",
            "tokensPerSec": 430,
            "ttftMs": 374
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-dev-craftbook-routing",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "dev-craftbook-routing",
        "repetition": 2
      },
      "trialId": "dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-30T14-58-24-158Z-43sc",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-30T14-58-24-158Z-43sc/recording/chat-events.jsonl",
      "recordingSha256": "ba1bca741651b2dbf45d946676d807a4f55bddb4aaa6a11ebd4f8542198b854e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-10-30T14-58-24-158Z-43sc/metrics.json",
      "metricsSha256": "e74c6b3e633652b81629f2ca5cd1fb3a87f4963f7b496a79cc14499e9155bbbd",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 487,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-10-30T14:58:56.884Z",
            "sessionId": "41714124-0a86-42c5-9d5b-f8a721a588ba",
            "source": "first-token-detail",
            "tokensPerSec": 487,
            "ttftMs": 30995
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-interface-contract",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "interface-contract",
        "repetition": 2
      },
      "trialId": "interface-contract-mlx-qwen3-8-27b-q4-2026-10-30T15-04-01-145Z-6gdo",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-qwen3-8-27b-q4-2026-10-30T15-04-01-145Z-6gdo/recording/chat-events.jsonl",
      "recordingSha256": "8c2ef451cf2848460240849771794945256b6d7d39a4fa23b0a18acf6f0f4517",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-qwen3-8-27b-q4-2026-10-30T15-04-01-145Z-6gdo/metrics.json",
      "metricsSha256": "42e04c9138ef33b2ccbead9290e05ffbad95ca700fc8956708296f57743e61e9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 735.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-30T15:04:25.294Z",
            "sessionId": "82beaed3-d361-4a3b-8dbb-335ca95c596c",
            "source": "first-token-detail",
            "tokensPerSec": 758,
            "ttftMs": 22461
          },
          {
            "line": 191,
            "rx": "2026-10-30T15:04:56.319Z",
            "sessionId": "7438a4e1-7881-48b7-87f7-be22e0ff4eb5",
            "source": "first-token-detail",
            "tokensPerSec": 713,
            "ttftMs": 7537
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-interface-contract",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "interface-contract",
        "repetition": 2
      },
      "trialId": "interface-contract-mlx-gemma4-12b-q4-2026-10-30T15-07-39-504Z-xf4f",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-gemma4-12b-q4-2026-10-30T15-07-39-504Z-xf4f/recording/chat-events.jsonl",
      "recordingSha256": "d429885cc26cf3211ad04226c8cb54597bb7decfe6b2ff932d78472b0b40b903",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-gemma4-12b-q4-2026-10-30T15-07-39-504Z-xf4f/metrics.json",
      "metricsSha256": "b45332f112555205752980004ffb9ea813d102ec408f6448ce2429c748a87129",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1534.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 23,
            "rx": "2026-10-30T15:07:54.849Z",
            "sessionId": "309d1bb1-a9bd-4242-90e2-92355cf1e9d2",
            "source": "first-token-detail",
            "tokensPerSec": 1400,
            "ttftMs": 13649
          },
          {
            "line": 285,
            "rx": "2026-10-30T15:08:40.571Z",
            "sessionId": "7b5c77df-84d9-4a8f-904a-91f638819e5f",
            "source": "first-token-detail",
            "tokensPerSec": 1669,
            "ttftMs": 4448
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-craftbook-code-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-code-review",
        "repetition": 2
      },
      "trialId": "craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-30T15-09-19-166Z-t6c7",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-30T15-09-19-166Z-t6c7/recording/chat-events.jsonl",
      "recordingSha256": "ca30ff00e92ab2af8b187dab070d3d5b610f3601c5ee36d886506ed65bcab488",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-qwen3-8-27b-q4-2026-10-30T15-09-19-166Z-t6c7/metrics.json",
      "metricsSha256": "a91434a22d1148dbe76e566663321ae820cb513ca11bcc8edb0b44789e3e4d48",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 675,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T15:09:43.134Z",
            "sessionId": "06129994-fb69-411a-aa00-6a6d43053aa8",
            "source": "first-token-detail",
            "tokensPerSec": 668,
            "ttftMs": 17419
          },
          {
            "line": 285,
            "rx": "2026-10-30T15:10:42.383Z",
            "sessionId": "06129994-fb69-411a-aa00-6a6d43053aa8",
            "source": "first-token-detail",
            "tokensPerSec": 682,
            "ttftMs": 16582
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-craftbook-code-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-code-review",
        "repetition": 2
      },
      "trialId": "craftbook-code-review-mlx-gemma4-12b-q4-2026-10-30T15-19-57-027Z-3y0l",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-gemma4-12b-q4-2026-10-30T15-19-57-027Z-3y0l/recording/chat-events.jsonl",
      "recordingSha256": "6166aa50a11aacbe8553133b40f49df59af82b05a7c13f16b67cfeb8ee71c305",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-gemma4-12b-q4-2026-10-30T15-19-57-027Z-3y0l/metrics.json",
      "metricsSha256": "37ac75396d5d0d09a040058df241b7e53319443d4259b8f45551eed08a0dc31c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1239.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T15:20:16.767Z",
            "sessionId": "9af1c1dd-0149-472a-b5b5-94fc08b65dbf",
            "source": "first-token-detail",
            "tokensPerSec": 1187,
            "ttftMs": 13193
          },
          {
            "line": 107,
            "rx": "2026-10-30T15:20:41.921Z",
            "sessionId": "9af1c1dd-0149-472a-b5b5-94fc08b65dbf",
            "source": "first-token-detail",
            "tokensPerSec": 1292,
            "ttftMs": 8328
          },
          {
            "line": 1057,
            "rx": "2026-10-30T15:23:21.497Z",
            "sessionId": "9af1c1dd-0149-472a-b5b5-94fc08b65dbf",
            "source": "first-token-detail",
            "tokensPerSec": 1178,
            "ttftMs": 7871
          },
          {
            "line": 1252,
            "rx": "2026-10-30T15:23:59.005Z",
            "sessionId": "43fc75ab-f0c0-4335-b4ee-d9c50d0888c2",
            "source": "first-token-detail",
            "tokensPerSec": 1469,
            "ttftMs": 4408
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-craftbook-api-contract-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-api-contract-review",
        "repetition": 2
      },
      "trialId": "craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-30T15-47-33-441Z-51vg",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-30T15-47-33-441Z-51vg/recording/chat-events.jsonl",
      "recordingSha256": "0340295ee67463a7303f1255e76d241b7cc1c1cac8bfe15f27127361b2afb970",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-10-30T15-47-33-441Z-51vg/metrics.json",
      "metricsSha256": "837b65cb247bf35660372e6345e3046c9545558b27cfb6b9df3af850da7a74ae",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1104,
        "sampleCount": 15,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 15,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T15:47:54.078Z",
            "sessionId": "8b5845f2-027a-40eb-aa5f-4d2b77732838",
            "source": "first-token-detail",
            "tokensPerSec": 1127,
            "ttftMs": 14115
          },
          {
            "line": 403,
            "rx": "2026-10-30T15:49:20.794Z",
            "sessionId": "d124ec73-5750-45bc-bc80-2563ccb81939",
            "source": "first-token-detail",
            "tokensPerSec": 1443,
            "ttftMs": 5718
          },
          {
            "line": 665,
            "rx": "2026-10-30T15:50:24.643Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 1516,
            "ttftMs": 4586
          },
          {
            "line": 837,
            "rx": "2026-10-30T15:51:12.647Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 1308,
            "ttftMs": 7636
          },
          {
            "line": 1007,
            "rx": "2026-10-30T15:52:00.105Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 1229,
            "ttftMs": 10022
          },
          {
            "line": 2876,
            "rx": "2026-10-30T15:57:30.623Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 1104,
            "ttftMs": 20412
          },
          {
            "line": 3113,
            "rx": "2026-10-30T15:58:38.580Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 1035,
            "ttftMs": 23442
          },
          {
            "line": 4682,
            "rx": "2026-10-30T16:03:33.282Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 1026,
            "ttftMs": 32962
          },
          {
            "line": 4846,
            "rx": "2026-10-30T16:04:46.945Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 991,
            "ttftMs": 36698
          },
          {
            "line": 5038,
            "rx": "2026-10-30T16:05:16.937Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 622,
            "ttftMs": 752
          },
          {
            "line": 5351,
            "rx": "2026-10-30T16:06:54.456Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 971,
            "ttftMs": 39035
          },
          {
            "line": 5620,
            "rx": "2026-10-30T16:08:28.247Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 921,
            "ttftMs": 43706
          },
          {
            "line": 5885,
            "rx": "2026-10-30T16:09:10.521Z",
            "sessionId": "89ff97da-4385-465b-9762-e78e13a37d7d",
            "source": "first-token-detail",
            "tokensPerSec": 623,
            "ttftMs": 1017
          },
          {
            "line": 6142,
            "rx": "2026-10-30T16:10:07.280Z",
            "sessionId": "56c49ca0-d73d-4b55-860f-b550dd6ce142",
            "source": "first-token-detail",
            "tokensPerSec": 1231,
            "ttftMs": 6903
          },
          {
            "line": 6284,
            "rx": "2026-10-30T16:10:34.477Z",
            "sessionId": "f2b7916e-c008-4983-8d29-e246928cb932",
            "source": "first-token-detail",
            "tokensPerSec": 1243,
            "ttftMs": 3927
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-craftbook-api-contract-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-api-contract-review",
        "repetition": 2
      },
      "trialId": "craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-30T16-17-27-491Z-1jdy",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-30T16-17-27-491Z-1jdy/recording/chat-events.jsonl",
      "recordingSha256": "6f2d754e34e4fcd0b963ed4bc623f92ad9f1cf0fa768b0b826731179ea7df257",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-10-30T16-17-27-491Z-1jdy/metrics.json",
      "metricsSha256": "1d2f5c353db2983b978ab5dc4e0e25571f070508d84d46e244c87c29f4dbd067",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 570,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-30T16:17:55.939Z",
            "sessionId": "675b5e3e-5b46-4329-b3bc-b89ff2b25809",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 21926
          },
          {
            "line": 131,
            "rx": "2026-10-30T16:18:23.505Z",
            "sessionId": "675b5e3e-5b46-4329-b3bc-b89ff2b25809",
            "source": "first-token-detail",
            "tokensPerSec": 580,
            "ttftMs": 14345
          },
          {
            "line": 1786,
            "rx": "2026-10-30T16:23:33.146Z",
            "sessionId": "3f021f10-9a69-40cd-9a3b-dc9da47b4a1b",
            "source": "first-token-detail",
            "tokensPerSec": 601,
            "ttftMs": 13997
          },
          {
            "line": 6977,
            "rx": "2026-10-30T16:39:48.574Z",
            "sessionId": "1b0d0294-e5ca-4834-9f9b-0a81d0672f73",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 9047
          },
          {
            "line": 9220,
            "rx": "2026-10-30T16:47:13.974Z",
            "sessionId": "1b0d0294-e5ca-4834-9f9b-0a81d0672f73",
            "source": "first-token-detail",
            "tokensPerSec": 509,
            "ttftMs": 29308
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-craftbook-deep-security-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-deep-security-review",
        "repetition": 2
      },
      "trialId": "craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-30T16-48-53-614Z-mfkk",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-30T16-48-53-614Z-mfkk/recording/chat-events.jsonl",
      "recordingSha256": "0028cd510d4e9fa92cfbefbf868b8b68fa441df0105b98d846850ce5c494ef16",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-10-30T16-48-53-614Z-mfkk/metrics.json",
      "metricsSha256": "ab97e8accbed070fe80e125c162890630780f8ce93b1724f901b34272dee7bd2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1163.5,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-30T16:49:08.825Z",
            "sessionId": "1c8e7bec-618a-4879-a2bf-79965e6ff7d2",
            "source": "first-token-detail",
            "tokensPerSec": 1111,
            "ttftMs": 13571
          },
          {
            "line": 207,
            "rx": "2026-10-30T16:50:07.521Z",
            "sessionId": "1c8e7bec-618a-4879-a2bf-79965e6ff7d2",
            "source": "first-token-detail",
            "tokensPerSec": 1281,
            "ttftMs": 6897
          },
          {
            "line": 332,
            "rx": "2026-10-30T16:50:47.089Z",
            "sessionId": "1c8e7bec-618a-4879-a2bf-79965e6ff7d2",
            "source": "first-token-detail",
            "tokensPerSec": 1331,
            "ttftMs": 933
          },
          {
            "line": 399,
            "rx": "2026-10-30T16:51:07.595Z",
            "sessionId": "1c8e7bec-618a-4879-a2bf-79965e6ff7d2",
            "source": "first-token-detail",
            "tokensPerSec": 1216,
            "ttftMs": 1179
          },
          {
            "line": 428,
            "rx": "2026-10-30T16:51:33.194Z",
            "sessionId": "1c8e7bec-618a-4879-a2bf-79965e6ff7d2",
            "source": "first-token-detail",
            "tokensPerSec": 1101,
            "ttftMs": 11594
          },
          {
            "line": 572,
            "rx": "2026-10-30T16:52:31.701Z",
            "sessionId": "1c8e7bec-618a-4879-a2bf-79965e6ff7d2",
            "source": "first-token-detail",
            "tokensPerSec": 1066,
            "ttftMs": 14469
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-craftbook-deep-security-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-deep-security-review",
        "repetition": 2
      },
      "trialId": "craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-30T16-59-20-292Z-aad5",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-30T16-59-20-292Z-aad5/recording/chat-events.jsonl",
      "recordingSha256": "a405b312408be16e0c9cd4cefca54ad71dd8346210a7ed6a31f38a4d0bbdb26a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-10-30T16-59-20-292Z-aad5/metrics.json",
      "metricsSha256": "2d005288c0ee33c34c4d29b7303acb81d7f26733a7afc5a2a7d68b3437925591",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 502,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T16:59:44.999Z",
            "sessionId": "e7c99fb0-47f4-47fd-9f06-7a3b45ebb823",
            "source": "first-token-detail",
            "tokensPerSec": 522,
            "ttftMs": 23057
          },
          {
            "line": 2601,
            "rx": "2026-10-30T17:07:35.917Z",
            "sessionId": "4d8fb1c8-ba40-4919-9577-2d14f625a82a",
            "source": "first-token-detail",
            "tokensPerSec": 482,
            "ttftMs": 9919
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-large-pr-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "large-pr-review",
        "repetition": 2
      },
      "trialId": "large-pr-review-mlx-qwen3-8-27b-q4-2026-10-30T17-42-15-792Z-imul",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-qwen3-8-27b-q4-2026-10-30T17-42-15-792Z-imul/recording/chat-events.jsonl",
      "recordingSha256": "1eb394f846b6f8a87720356d40be196eb904696d2de56000b07f6385b2b16abd",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-qwen3-8-27b-q4-2026-10-30T17-42-15-792Z-imul/metrics.json",
      "metricsSha256": "62d65019a605402b77c60cb82eff7b240d3ac57b5dcee9b5cae8dfff48f253d0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 155,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-10-30T17:42:50.599Z",
            "sessionId": "3e55db7d-42eb-4d86-8273-3d92ad28a458",
            "source": "first-token-detail",
            "tokensPerSec": 532,
            "ttftMs": 33018
          },
          {
            "line": 10594,
            "rx": "2026-10-30T18:18:07.527Z",
            "sessionId": "3e55db7d-42eb-4d86-8273-3d92ad28a458",
            "source": "first-token-detail",
            "tokensPerSec": 155,
            "ttftMs": 4741
          },
          {
            "line": 13347,
            "rx": "2026-10-30T18:26:26.934Z",
            "sessionId": "3e55db7d-42eb-4d86-8273-3d92ad28a458",
            "source": "first-token-detail",
            "tokensPerSec": 134,
            "ttftMs": 2559
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-large-pr-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "large-pr-review",
        "repetition": 2
      },
      "trialId": "large-pr-review-mlx-gemma4-12b-q4-2026-10-30T19-02-22-314Z-2sgr",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-gemma4-12b-q4-2026-10-30T19-02-22-314Z-2sgr/recording/chat-events.jsonl",
      "recordingSha256": "732682b89127053ad8b297ef93a8fc70d3feea307783966c77c59e4bd69dd2a5",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-gemma4-12b-q4-2026-10-30T19-02-22-314Z-2sgr/metrics.json",
      "metricsSha256": "156c77812785ccf44b926cc1df09a324c96b377407ac3d7f1c2f3de6f616a1a7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 622,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-30T19:02:42.989Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 938,
            "ttftMs": 18895
          },
          {
            "line": 2004,
            "rx": "2026-10-30T19:10:14.102Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 813,
            "ttftMs": 59066
          },
          {
            "line": 2080,
            "rx": "2026-10-30T19:11:36.328Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 760,
            "ttftMs": 70474
          },
          {
            "line": 2276,
            "rx": "2026-10-30T19:12:14.674Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 421,
            "ttftMs": 684
          },
          {
            "line": 3615,
            "rx": "2026-10-30T19:17:40.189Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 831,
            "ttftMs": 63413
          },
          {
            "line": 3671,
            "rx": "2026-10-30T19:18:54.148Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 798,
            "ttftMs": 68320
          },
          {
            "line": 4092,
            "rx": "2026-10-30T19:21:09.873Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 452,
            "ttftMs": 1705
          },
          {
            "line": 4506,
            "rx": "2026-10-30T19:23:15.345Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 463,
            "ttftMs": 1666
          },
          {
            "line": 5006,
            "rx": "2026-10-30T19:26:05.187Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 429,
            "ttftMs": 1806
          },
          {
            "line": 5533,
            "rx": "2026-10-30T19:30:55.043Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 649,
            "ttftMs": 135754
          },
          {
            "line": 5663,
            "rx": "2026-10-30T19:35:46.103Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 622,
            "ttftMs": 144817
          },
          {
            "line": 6529,
            "rx": "2026-10-30T19:40:36.128Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 302,
            "ttftMs": 990
          },
          {
            "line": 7144,
            "rx": "2026-10-30T19:42:17.926Z",
            "sessionId": "f4eff8c9-af4d-45c2-b94b-6ba398da41e7",
            "source": "first-token-detail",
            "tokensPerSec": 301,
            "ttftMs": 824
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-craftbook-codemod-sweep",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-codemod-sweep",
        "repetition": 2
      },
      "trialId": "craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-30T19-42-42-401Z-xq97",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-30T19-42-42-401Z-xq97/recording/chat-events.jsonl",
      "recordingSha256": "bc41a26d233120114f578ef5a27c225388010dcf2a173b68a8aeb6620cfe309f",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-10-30T19-42-42-401Z-xq97/metrics.json",
      "metricsSha256": "ddd60afadf81bb2a631711e29b14e23683e048eeb9d48eaf6b3770cdfd8fcc02",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 477,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-30T19:43:10.464Z",
            "sessionId": "a171a38e-1037-4b52-a215-f81c5e3f4cf4",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 21476
          },
          {
            "line": 1691,
            "rx": "2026-10-30T19:48:11.278Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 17105
          },
          {
            "line": 2380,
            "rx": "2026-10-30T19:50:37.461Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 532,
            "ttftMs": 18238
          },
          {
            "line": 3535,
            "rx": "2026-10-30T19:54:15.032Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 226,
            "ttftMs": 980
          },
          {
            "line": 3755,
            "rx": "2026-10-30T19:55:17.895Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 496,
            "ttftMs": 21463
          },
          {
            "line": 4100,
            "rx": "2026-10-30T19:56:22.223Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 319,
            "ttftMs": 1080
          },
          {
            "line": 4542,
            "rx": "2026-10-30T19:57:42.392Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 361,
            "ttftMs": 908
          },
          {
            "line": 4779,
            "rx": "2026-10-30T19:58:22.584Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 937
          },
          {
            "line": 5404,
            "rx": "2026-10-30T20:00:49.300Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 478,
            "ttftMs": 35109
          },
          {
            "line": 6414,
            "rx": "2026-10-30T20:04:17.793Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 30843
          },
          {
            "line": 6756,
            "rx": "2026-10-30T20:05:48.507Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 479,
            "ttftMs": 32539
          },
          {
            "line": 7252,
            "rx": "2026-10-30T20:07:52.554Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 466,
            "ttftMs": 35155
          },
          {
            "line": 9225,
            "rx": "2026-10-30T20:14:33.747Z",
            "sessionId": "dfd8b59a-92cf-4d1d-bdcc-9e9c7b4054e1",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 40295
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-craftbook-codemod-sweep",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-codemod-sweep",
        "repetition": 2
      },
      "trialId": "craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-30T20-56-48-802Z-w08m",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-30T20-56-48-802Z-w08m/recording/chat-events.jsonl",
      "recordingSha256": "0deb17c0434df3c87522148c86ae519304506459ecb94bea2eafc4cfe2a55993",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-10-30T20-56-48-802Z-w08m/metrics.json",
      "metricsSha256": "85744b7ed5ad874fb11b70a4500d2c0fb71b9f81706ac30790a07914e20e5cec",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1036,
        "sampleCount": 14,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 14,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T20:57:09.152Z",
            "sessionId": "14e7d1bf-76f5-48ad-a1ab-9d94e0ac6ac3",
            "source": "first-token-detail",
            "tokensPerSec": 1093,
            "ttftMs": 13827
          },
          {
            "line": 149,
            "rx": "2026-10-30T20:57:42.535Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1289,
            "ttftMs": 7190
          },
          {
            "line": 416,
            "rx": "2026-10-30T20:58:42.887Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1123,
            "ttftMs": 10403
          },
          {
            "line": 521,
            "rx": "2026-10-30T20:59:00.960Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 884,
            "ttftMs": 901
          },
          {
            "line": 758,
            "rx": "2026-10-30T20:59:51.401Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1172,
            "ttftMs": 11014
          },
          {
            "line": 988,
            "rx": "2026-10-30T21:00:29.797Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 535,
            "ttftMs": 708
          },
          {
            "line": 1586,
            "rx": "2026-10-30T21:02:18.425Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1060,
            "ttftMs": 11766
          },
          {
            "line": 3620,
            "rx": "2026-10-30T21:08:08.009Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1032,
            "ttftMs": 14525
          },
          {
            "line": 4007,
            "rx": "2026-10-30T21:09:28.824Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 983,
            "ttftMs": 15973
          },
          {
            "line": 4597,
            "rx": "2026-10-30T21:11:21.896Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1061,
            "ttftMs": 15486
          },
          {
            "line": 4981,
            "rx": "2026-10-30T21:12:30.688Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 915,
            "ttftMs": 1579
          },
          {
            "line": 5431,
            "rx": "2026-10-30T21:14:09.032Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 1040,
            "ttftMs": 1323
          },
          {
            "line": 5819,
            "rx": "2026-10-30T21:15:17.453Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 916,
            "ttftMs": 2398
          },
          {
            "line": 6179,
            "rx": "2026-10-30T21:16:23.417Z",
            "sessionId": "67e79a32-c071-4593-81fb-17928f255aed",
            "source": "first-token-detail",
            "tokensPerSec": 967,
            "ttftMs": 709
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-craftbook-refactor-module",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-refactor-module",
        "repetition": 2
      },
      "trialId": "craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-30T22-25-32-724Z-5c3y",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-30T22-25-32-724Z-5c3y/recording/chat-events.jsonl",
      "recordingSha256": "d5f27d275b83edb9c229fa9cba516d6dc4d742c3e8c05f558060821f1dcb05a4",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-gemma4-12b-q4-2026-10-30T22-25-32-724Z-5c3y/metrics.json",
      "metricsSha256": "2bc301c2051ebcf9d9ca731b34db61d8969954f5963fe256161c64e18d24327d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1105.5,
        "sampleCount": 36,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 36,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-30T22:25:50.078Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1557,
            "ttftMs": 10835
          },
          {
            "line": 64,
            "rx": "2026-10-30T22:25:58.570Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1391,
            "ttftMs": 956
          },
          {
            "line": 98,
            "rx": "2026-10-30T22:26:06.987Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1277,
            "ttftMs": 1038
          },
          {
            "line": 140,
            "rx": "2026-10-30T22:26:22.558Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1505,
            "ttftMs": 7640
          },
          {
            "line": 164,
            "rx": "2026-10-30T22:26:37.927Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1593,
            "ttftMs": 8711
          },
          {
            "line": 537,
            "rx": "2026-10-30T22:27:45.651Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1139,
            "ttftMs": 10671
          },
          {
            "line": 575,
            "rx": "2026-10-30T22:27:56.348Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 921,
            "ttftMs": 1428
          },
          {
            "line": 624,
            "rx": "2026-10-30T22:28:20.244Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1086,
            "ttftMs": 13794
          },
          {
            "line": 669,
            "rx": "2026-10-30T22:30:44.351Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1465,
            "ttftMs": 11156
          },
          {
            "line": 701,
            "rx": "2026-10-30T22:30:54.298Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1199,
            "ttftMs": 824
          },
          {
            "line": 754,
            "rx": "2026-10-30T22:31:15.652Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1441,
            "ttftMs": 14438
          },
          {
            "line": 1090,
            "rx": "2026-10-30T22:32:19.462Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1301,
            "ttftMs": 15034
          },
          {
            "line": 1123,
            "rx": "2026-10-30T22:32:43.359Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1239,
            "ttftMs": 18850
          },
          {
            "line": 1410,
            "rx": "2026-10-30T22:33:49.468Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 985,
            "ttftMs": 22595
          },
          {
            "line": 1457,
            "rx": "2026-10-30T22:34:03.193Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 630,
            "ttftMs": 2076
          },
          {
            "line": 1520,
            "rx": "2026-10-30T22:35:47.361Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1035,
            "ttftMs": 24543
          },
          {
            "line": 1558,
            "rx": "2026-10-30T22:35:58.471Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1086,
            "ttftMs": 753
          },
          {
            "line": 1620,
            "rx": "2026-10-30T22:36:57.779Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1101,
            "ttftMs": 24592
          },
          {
            "line": 1672,
            "rx": "2026-10-30T22:39:14.472Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1456,
            "ttftMs": 19302
          },
          {
            "line": 1697,
            "rx": "2026-10-30T22:39:21.195Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1240,
            "ttftMs": 705
          },
          {
            "line": 1739,
            "rx": "2026-10-30T22:40:22.047Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1014,
            "ttftMs": 640
          },
          {
            "line": 1774,
            "rx": "2026-10-30T22:41:33.313Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1110,
            "ttftMs": 819
          },
          {
            "line": 1808,
            "rx": "2026-10-30T22:41:43.574Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1167,
            "ttftMs": 928
          },
          {
            "line": 1840,
            "rx": "2026-10-30T22:42:54.623Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1083,
            "ttftMs": 887
          },
          {
            "line": 1861,
            "rx": "2026-10-30T22:44:00.446Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1088,
            "ttftMs": 736
          },
          {
            "line": 1882,
            "rx": "2026-10-30T22:45:06.407Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 999,
            "ttftMs": 697
          },
          {
            "line": 1907,
            "rx": "2026-10-30T22:45:11.436Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1165,
            "ttftMs": 656
          },
          {
            "line": 1932,
            "rx": "2026-10-30T22:45:16.812Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 995,
            "ttftMs": 943
          },
          {
            "line": 1957,
            "rx": "2026-10-30T22:46:27.620Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1005,
            "ttftMs": 717
          },
          {
            "line": 1989,
            "rx": "2026-10-30T22:46:37.916Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1011,
            "ttftMs": 864
          },
          {
            "line": 2034,
            "rx": "2026-10-30T22:47:17.002Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1301,
            "ttftMs": 29210
          },
          {
            "line": 2625,
            "rx": "2026-10-30T22:49:49.292Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 965,
            "ttftMs": 43926
          },
          {
            "line": 2686,
            "rx": "2026-10-30T22:51:50.483Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 942,
            "ttftMs": 47031
          },
          {
            "line": 2719,
            "rx": "2026-10-30T22:53:01.821Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 829
          },
          {
            "line": 2749,
            "rx": "2026-10-30T22:53:12.275Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 993
          },
          {
            "line": 2837,
            "rx": "2026-10-30T22:54:41.495Z",
            "sessionId": "d75dcf8e-4670-41ef-b7d4-de61caa3ae1b",
            "source": "first-token-detail",
            "tokensPerSec": 1166,
            "ttftMs": 38903
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-craftbook-refactor-module",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-refactor-module",
        "repetition": 2
      },
      "trialId": "craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-30T23-22-29-040Z-ku0t",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-30T23-22-29-040Z-ku0t/recording/chat-events.jsonl",
      "recordingSha256": "25c557d4629a83b8bd3bb44725a9fd5ce84d93c6072a0cf1438baa2f77754ef9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-10-30T23-22-29-040Z-ku0t/metrics.json",
      "metricsSha256": "dbe8ddc27b6b48509b9b0602fe2fc259acc1a2aa4798bc918b1e194956d2b392",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 463,
        "sampleCount": 24,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 24,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-30T23:22:51.564Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 16008
          },
          {
            "line": 142,
            "rx": "2026-10-30T23:23:10.568Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 506,
            "ttftMs": 1846
          },
          {
            "line": 275,
            "rx": "2026-10-30T23:23:33.202Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 472,
            "ttftMs": 1971
          },
          {
            "line": 391,
            "rx": "2026-10-30T23:24:11.114Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 589,
            "ttftMs": 20168
          },
          {
            "line": 699,
            "rx": "2026-10-30T23:25:06.438Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 761
          },
          {
            "line": 2148,
            "rx": "2026-10-30T23:29:24.000Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 236,
            "ttftMs": 1063
          },
          {
            "line": 2524,
            "rx": "2026-10-30T23:30:31.683Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 234,
            "ttftMs": 1293
          },
          {
            "line": 2978,
            "rx": "2026-10-30T23:32:16.701Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 26406
          },
          {
            "line": 3133,
            "rx": "2026-10-30T23:32:43.935Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 410,
            "ttftMs": 1331
          },
          {
            "line": 3210,
            "rx": "2026-10-30T23:32:58.687Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 354,
            "ttftMs": 866
          },
          {
            "line": 3341,
            "rx": "2026-10-30T23:34:07.631Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 24257
          },
          {
            "line": 3420,
            "rx": "2026-10-30T23:35:09.319Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 506,
            "ttftMs": 25209
          },
          {
            "line": 3534,
            "rx": "2026-10-30T23:38:55.675Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 774,
            "ttftMs": 18070
          },
          {
            "line": 3682,
            "rx": "2026-10-30T23:42:14.394Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 657,
            "ttftMs": 3568
          },
          {
            "line": 3787,
            "rx": "2026-10-30T23:42:45.285Z",
            "sessionId": "e6f6e950-e75a-4c1a-b064-3bb902996359",
            "source": "first-token-detail",
            "tokensPerSec": 454,
            "ttftMs": 36327
          },
          {
            "line": 4789,
            "rx": "2026-10-30T23:45:31.148Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 276,
            "ttftMs": 2848
          },
          {
            "line": 5351,
            "rx": "2026-10-30T23:47:17.405Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 235,
            "ttftMs": 2594
          },
          {
            "line": 5781,
            "rx": "2026-10-30T23:48:33.890Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 227,
            "ttftMs": 2047
          },
          {
            "line": 6464,
            "rx": "2026-10-30T23:50:38.168Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 249,
            "ttftMs": 2198
          },
          {
            "line": 7029,
            "rx": "2026-10-30T23:52:21.103Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 266,
            "ttftMs": 2140
          },
          {
            "line": 7595,
            "rx": "2026-10-30T23:54:03.884Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 256,
            "ttftMs": 1903
          },
          {
            "line": 8127,
            "rx": "2026-10-30T23:55:36.899Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 227,
            "ttftMs": 2204
          },
          {
            "line": 8373,
            "rx": "2026-10-30T23:57:31.127Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 481,
            "ttftMs": 39135
          },
          {
            "line": 8508,
            "rx": "2026-10-30T23:59:06.484Z",
            "sessionId": "7a1dc72b-4aef-49a5-be28-bc034a0b86f8",
            "source": "first-token-detail",
            "tokensPerSec": 474,
            "ttftMs": 42126
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-developer-craftbook-bug-fix-tdd",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-bug-fix-tdd",
        "repetition": 2
      },
      "trialId": "craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-31T00-00-32-268Z-0xx7",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-31T00-00-32-268Z-0xx7/recording/chat-events.jsonl",
      "recordingSha256": "3a20212dbb16faf26f11bc048759da358a7cc286f20c9633c2cb4a34f780e2a8",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-10-31T00-00-32-268Z-0xx7/metrics.json",
      "metricsSha256": "3bdd0eb6a1bc16f39b54e765908b70cf5050297c1b6e04bb3171d6616b24cf8a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1069,
        "sampleCount": 23,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 23,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T00:00:53.860Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1081,
            "ttftMs": 15062
          },
          {
            "line": 109,
            "rx": "2026-10-31T00:01:19.806Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1122,
            "ttftMs": 8736
          },
          {
            "line": 135,
            "rx": "2026-10-31T00:01:26.054Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 895,
            "ttftMs": 1148
          },
          {
            "line": 176,
            "rx": "2026-10-31T00:01:43.878Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1175,
            "ttftMs": 9809
          },
          {
            "line": 555,
            "rx": "2026-10-31T00:02:44.295Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1069,
            "ttftMs": 466
          },
          {
            "line": 1238,
            "rx": "2026-10-31T00:04:45.829Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1071,
            "ttftMs": 11834
          },
          {
            "line": 1266,
            "rx": "2026-10-31T00:04:54.534Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1103,
            "ttftMs": 719
          },
          {
            "line": 1291,
            "rx": "2026-10-31T00:05:14.611Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1068,
            "ttftMs": 15234
          },
          {
            "line": 1687,
            "rx": "2026-10-31T00:06:36.561Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1034,
            "ttftMs": 15424
          },
          {
            "line": 1720,
            "rx": "2026-10-31T00:06:46.949Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1035,
            "ttftMs": 833
          },
          {
            "line": 1747,
            "rx": "2026-10-31T00:07:09.593Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1009,
            "ttftMs": 17865
          },
          {
            "line": 2350,
            "rx": "2026-10-31T00:09:04.434Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 957,
            "ttftMs": 20192
          },
          {
            "line": 2375,
            "rx": "2026-10-31T00:09:15.017Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1082,
            "ttftMs": 713
          },
          {
            "line": 2403,
            "rx": "2026-10-31T00:09:42.368Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 948,
            "ttftMs": 22437
          },
          {
            "line": 2789,
            "rx": "2026-10-31T00:11:12.589Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 937,
            "ttftMs": 25090
          },
          {
            "line": 2835,
            "rx": "2026-10-31T00:11:48.078Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 998,
            "ttftMs": 24732
          },
          {
            "line": 2865,
            "rx": "2026-10-31T00:12:48.549Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1036,
            "ttftMs": 23975
          },
          {
            "line": 2977,
            "rx": "2026-10-31T00:15:25.082Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1501,
            "ttftMs": 17825
          },
          {
            "line": 3005,
            "rx": "2026-10-31T00:16:34.873Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1105,
            "ttftMs": 758
          },
          {
            "line": 3029,
            "rx": "2026-10-31T00:17:41.218Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1070,
            "ttftMs": 687
          },
          {
            "line": 3066,
            "rx": "2026-10-31T00:18:12.223Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1449,
            "ttftMs": 20891
          },
          {
            "line": 3241,
            "rx": "2026-10-31T00:19:40.776Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1403,
            "ttftMs": 22674
          },
          {
            "line": 3282,
            "rx": "2026-10-31T00:19:55.123Z",
            "sessionId": "62092344-df74-44b2-8588-794dd2818a2f",
            "source": "first-token-detail",
            "tokensPerSec": 1043,
            "ttftMs": 1870
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-developer-craftbook-bug-fix-tdd",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-bug-fix-tdd",
        "repetition": 2
      },
      "trialId": "craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-31T01-01-49-395Z-ikz8",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-31T01-01-49-395Z-ikz8/recording/chat-events.jsonl",
      "recordingSha256": "7b979704b0af7e29173c0946520c12da296edada02c13b0cca253a30805f0143",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-10-31T01-01-49-395Z-ikz8/metrics.json",
      "metricsSha256": "81227d4d4aa54533f8d05f4b9cf86623470aaf64156c1edf396ae621778e2444",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 477,
        "sampleCount": 23,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 23,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-10-31T01:02:20.422Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 506,
            "ttftMs": 24478
          },
          {
            "line": 1089,
            "rx": "2026-10-31T01:05:16.929Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 248,
            "ttftMs": 957
          },
          {
            "line": 1957,
            "rx": "2026-10-31T01:08:14.362Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 22118
          },
          {
            "line": 2344,
            "rx": "2026-10-31T01:09:23.081Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 298,
            "ttftMs": 1100
          },
          {
            "line": 2639,
            "rx": "2026-10-31T01:10:35.767Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 518,
            "ttftMs": 20451
          },
          {
            "line": 2708,
            "rx": "2026-10-31T01:11:19.903Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 489,
            "ttftMs": 17085
          },
          {
            "line": 4761,
            "rx": "2026-10-31T01:17:38.403Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 226,
            "ttftMs": 1103
          },
          {
            "line": 5283,
            "rx": "2026-10-31T01:19:36.619Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 480,
            "ttftMs": 27922
          },
          {
            "line": 5352,
            "rx": "2026-10-31T01:20:53.028Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 492,
            "ttftMs": 41630
          },
          {
            "line": 7624,
            "rx": "2026-10-31T01:27:43.369Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 206,
            "ttftMs": 1079
          },
          {
            "line": 8016,
            "rx": "2026-10-31T01:29:28.435Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 36976
          },
          {
            "line": 8728,
            "rx": "2026-10-31T01:31:37.468Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 190,
            "ttftMs": 1300
          },
          {
            "line": 8984,
            "rx": "2026-10-31T01:33:04.356Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 39385
          },
          {
            "line": 9154,
            "rx": "2026-10-31T01:33:33.610Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 429,
            "ttftMs": 1282
          },
          {
            "line": 9291,
            "rx": "2026-10-31T01:33:59.429Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 337,
            "ttftMs": 936
          },
          {
            "line": 9361,
            "rx": "2026-10-31T01:35:33.559Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 492,
            "ttftMs": 42986
          },
          {
            "line": 9477,
            "rx": "2026-10-31T01:37:19.151Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 471,
            "ttftMs": 46769
          },
          {
            "line": 9686,
            "rx": "2026-10-31T01:37:55.209Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 462,
            "ttftMs": 1287
          },
          {
            "line": 9804,
            "rx": "2026-10-31T01:39:49.229Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 49098
          },
          {
            "line": 10008,
            "rx": "2026-10-31T01:42:00.798Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 712,
            "ttftMs": 37858
          },
          {
            "line": 10065,
            "rx": "2026-10-31T01:44:23.587Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 686,
            "ttftMs": 36342
          },
          {
            "line": 10177,
            "rx": "2026-10-31T01:46:02.571Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 559,
            "ttftMs": 46432
          },
          {
            "line": 10301,
            "rx": "2026-10-31T01:48:24.902Z",
            "sessionId": "b1d3b264-c11d-4a3a-8e34-80dfc491572e",
            "source": "first-token-detail",
            "tokensPerSec": 370,
            "ttftMs": 73442
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-find-vs-create",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-find-vs-create",
        "repetition": 2
      },
      "trialId": "craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-31T04-09-10-016Z-b5qq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-31T04-09-10-016Z-b5qq/recording/chat-events.jsonl",
      "recordingSha256": "7cea9c869a52386319224b42932604c0b9e3ca80c45dcf6810b552f82b328fca",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-10-31T04-09-10-016Z-b5qq/metrics.json",
      "metricsSha256": "66a5c426efc7c7715f54d4930775bdc3ee0bece0ae870db06e7d79cb2dc49335",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 572,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T04:09:37.538Z",
            "sessionId": "adb18140-4680-4682-959c-bc0e0fb3cf62",
            "source": "first-token-detail",
            "tokensPerSec": 572,
            "ttftMs": 25832
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-find-vs-create",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-find-vs-create",
        "repetition": 2
      },
      "trialId": "craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-31T04-12-55-808Z-l1jn",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-31T04-12-55-808Z-l1jn/recording/chat-events.jsonl",
      "recordingSha256": "e2bf5e53d1c675e77ffdf5b5a6b07f80f9bfbafda4a9c7e1e3093c81b07c6184",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-10-31T04-12-55-808Z-l1jn/metrics.json",
      "metricsSha256": "39d3dc7204f603fc84a907df232dd17d92f3cd0c9fc1e2f54f0e8a05e2e15245",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1011,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-31T04:13:10.873Z",
            "sessionId": "ce6b51a6-6c05-43dc-a49a-e8a290a34d77",
            "source": "first-token-detail",
            "tokensPerSec": 1254,
            "ttftMs": 13382
          },
          {
            "line": 1522,
            "rx": "2026-10-31T04:17:20.212Z",
            "sessionId": "ce6b51a6-6c05-43dc-a49a-e8a290a34d77",
            "source": "first-token-detail",
            "tokensPerSec": 228,
            "ttftMs": 453
          },
          {
            "line": 1810,
            "rx": "2026-10-31T04:18:15.692Z",
            "sessionId": "0cf0debb-0eed-49de-9dba-6794e53f208e",
            "source": "first-token-detail",
            "tokensPerSec": 1028,
            "ttftMs": 11034
          },
          {
            "line": 1950,
            "rx": "2026-10-31T04:18:46.963Z",
            "sessionId": "ce6b51a6-6c05-43dc-a49a-e8a290a34d77",
            "source": "first-token-detail",
            "tokensPerSec": 591,
            "ttftMs": 596
          },
          {
            "line": 2021,
            "rx": "2026-10-31T04:19:02.875Z",
            "sessionId": "0cf0debb-0eed-49de-9dba-6794e53f208e",
            "source": "first-token-detail",
            "tokensPerSec": 1011,
            "ttftMs": 7486
          },
          {
            "line": 2086,
            "rx": "2026-10-31T04:19:24.941Z",
            "sessionId": "ce6b51a6-6c05-43dc-a49a-e8a290a34d77",
            "source": "first-token-detail",
            "tokensPerSec": 653,
            "ttftMs": 482
          },
          {
            "line": 2340,
            "rx": "2026-10-31T04:20:11.249Z",
            "sessionId": "0cf0debb-0eed-49de-9dba-6794e53f208e",
            "source": "first-token-detail",
            "tokensPerSec": 1011,
            "ttftMs": 7785
          },
          {
            "line": 2380,
            "rx": "2026-10-31T04:20:16.918Z",
            "sessionId": "ce6b51a6-6c05-43dc-a49a-e8a290a34d77",
            "source": "first-token-detail",
            "tokensPerSec": 608,
            "ttftMs": 680
          },
          {
            "line": 2619,
            "rx": "2026-10-31T04:20:59.101Z",
            "sessionId": "0cf0debb-0eed-49de-9dba-6794e53f208e",
            "source": "first-token-detail",
            "tokensPerSec": 1029,
            "ttftMs": 7154
          },
          {
            "line": 2674,
            "rx": "2026-10-31T04:21:23.742Z",
            "sessionId": "0cf0debb-0eed-49de-9dba-6794e53f208e",
            "source": "first-token-detail",
            "tokensPerSec": 1205,
            "ttftMs": 7867
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-edit-midtask",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-edit-midtask",
        "repetition": 2
      },
      "trialId": "craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-31T04-22-03-008Z-t82e",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-31T04-22-03-008Z-t82e/recording/chat-events.jsonl",
      "recordingSha256": "161ef28a47c6f7db9e6a8f38a465c0ab550b2d7337b307b247f8c181e64ccea3",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-10-31T04-22-03-008Z-t82e/metrics.json",
      "metricsSha256": "0923d254762448be673f24c44ab5ea41b63dbcccf32850d3fc052936b959c9a4",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 305,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 39,
            "rx": "2026-10-31T04:22:42.998Z",
            "sessionId": "3e694374-56e8-4628-8137-ee051cea39ac",
            "source": "first-token-detail",
            "tokensPerSec": 562,
            "ttftMs": 38312
          },
          {
            "line": 7985,
            "rx": "2026-10-31T04:46:48.798Z",
            "sessionId": "3e694374-56e8-4628-8137-ee051cea39ac",
            "source": "first-token-detail",
            "tokensPerSec": 276,
            "ttftMs": 1915
          },
          {
            "line": 11594,
            "rx": "2026-10-31T04:57:38.068Z",
            "sessionId": "5c97b56c-22fe-4007-9329-b9f5ba4950ae",
            "source": "first-token-detail",
            "tokensPerSec": 515,
            "ttftMs": 24250
          },
          {
            "line": 11745,
            "rx": "2026-10-31T04:58:08.803Z",
            "sessionId": "3e694374-56e8-4628-8137-ee051cea39ac",
            "source": "first-token-detail",
            "tokensPerSec": 305,
            "ttftMs": 1575
          },
          {
            "line": 13072,
            "rx": "2026-10-31T05:02:03.273Z",
            "sessionId": "5c97b56c-22fe-4007-9329-b9f5ba4950ae",
            "source": "first-token-detail",
            "tokensPerSec": 443,
            "ttftMs": 6926
          },
          {
            "line": 13187,
            "rx": "2026-10-31T05:02:27.760Z",
            "sessionId": "3e694374-56e8-4628-8137-ee051cea39ac",
            "source": "first-token-detail",
            "tokensPerSec": 203,
            "ttftMs": 1006
          },
          {
            "line": 13351,
            "rx": "2026-10-31T05:02:55.919Z",
            "sessionId": "3e694374-56e8-4628-8137-ee051cea39ac",
            "source": "first-token-detail",
            "tokensPerSec": 201,
            "ttftMs": 1003
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-edit-midtask",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-edit-midtask",
        "repetition": 2
      },
      "trialId": "craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-31T05-16-47-886Z-ymnx",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-31T05-16-47-886Z-ymnx/recording/chat-events.jsonl",
      "recordingSha256": "7ccff82a89f2d361faf8aaadb1380770b1d5241a8dbf2f39dfdae12e3a9470dd",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-10-31T05-16-47-886Z-ymnx/metrics.json",
      "metricsSha256": "8a089841b084deb2041d083812c6a4c5fad5cf06b1356b591a1dbac39802f849",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 962,
        "sampleCount": 15,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 15,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-31T05:17:08.891Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 1032,
            "ttftMs": 19433
          },
          {
            "line": 530,
            "rx": "2026-10-31T05:18:41.523Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 442,
            "ttftMs": 866
          },
          {
            "line": 795,
            "rx": "2026-10-31T05:19:24.155Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 490,
            "ttftMs": 671
          },
          {
            "line": 1154,
            "rx": "2026-10-31T05:20:25.803Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 411,
            "ttftMs": 860
          },
          {
            "line": 1590,
            "rx": "2026-10-31T05:21:44.787Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 515,
            "ttftMs": 1101
          },
          {
            "line": 1988,
            "rx": "2026-10-31T05:22:55.755Z",
            "sessionId": "94b3efc2-1e34-442f-ab68-1fcb2cb37a01",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 5234
          },
          {
            "line": 2082,
            "rx": "2026-10-31T05:23:12.432Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 478,
            "ttftMs": 648
          },
          {
            "line": 2358,
            "rx": "2026-10-31T05:23:58.957Z",
            "sessionId": "94b3efc2-1e34-442f-ab68-1fcb2cb37a01",
            "source": "first-token-detail",
            "tokensPerSec": 1011,
            "ttftMs": 7118
          },
          {
            "line": 2372,
            "rx": "2026-10-31T05:24:07.770Z",
            "sessionId": "6e88c393-ef6a-45d9-803c-4df70301ebfe",
            "source": "first-token-detail",
            "tokensPerSec": 1050,
            "ttftMs": 13305
          },
          {
            "line": 2539,
            "rx": "2026-10-31T05:24:35.211Z",
            "sessionId": "8e99323f-65a8-4651-9ab4-e4577be84aad",
            "source": "first-token-detail",
            "tokensPerSec": 486,
            "ttftMs": 635
          },
          {
            "line": 2568,
            "rx": "2026-10-31T05:24:42.104Z",
            "sessionId": "94b3efc2-1e34-442f-ab68-1fcb2cb37a01",
            "source": "first-token-detail",
            "tokensPerSec": 962,
            "ttftMs": 5793
          },
          {
            "line": 2637,
            "rx": "2026-10-31T05:25:00.942Z",
            "sessionId": "94b3efc2-1e34-442f-ab68-1fcb2cb37a01",
            "source": "first-token-detail",
            "tokensPerSec": 999,
            "ttftMs": 7404
          },
          {
            "line": 2680,
            "rx": "2026-10-31T05:25:22.806Z",
            "sessionId": "94b3efc2-1e34-442f-ab68-1fcb2cb37a01",
            "source": "first-token-detail",
            "tokensPerSec": 1013,
            "ttftMs": 14271
          },
          {
            "line": 2860,
            "rx": "2026-10-31T05:25:54.752Z",
            "sessionId": "6e88c393-ef6a-45d9-803c-4df70301ebfe",
            "source": "first-token-detail",
            "tokensPerSec": 1017,
            "ttftMs": 20049
          },
          {
            "line": 3152,
            "rx": "2026-10-31T05:26:38.579Z",
            "sessionId": "94b3efc2-1e34-442f-ab68-1fcb2cb37a01",
            "source": "first-token-detail",
            "tokensPerSec": 477,
            "ttftMs": 51211
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-invoice-run",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-invoice-run",
        "repetition": 2
      },
      "trialId": "craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-31T06-40-09-922Z-nrko",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-31T06-40-09-922Z-nrko/recording/chat-events.jsonl",
      "recordingSha256": "7e8156c9459cb29997fed94b922ee033da10780aead20afd2844bd34b9ec3884",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-gemma4-12b-q4-2026-10-31T06-40-09-922Z-nrko/metrics.json",
      "metricsSha256": "f7d6cb1c2c5fe5a34feb2f0e176a58b4e920e258e323901f297d565010aec0f6",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1058,
        "sampleCount": 12,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 12,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T06:40:30.360Z",
            "sessionId": "73b056e5-ee9c-4378-8cdc-2634d9d5348a",
            "source": "first-token-detail",
            "tokensPerSec": 1118,
            "ttftMs": 13907
          },
          {
            "line": 162,
            "rx": "2026-10-31T06:41:03.025Z",
            "sessionId": "6acd7240-b6a9-4b6c-9192-959e4f6701c4",
            "source": "first-token-detail",
            "tokensPerSec": 1283,
            "ttftMs": 6538
          },
          {
            "line": 340,
            "rx": "2026-10-31T06:41:57.370Z",
            "sessionId": "5b37b376-c9f4-4555-90a7-6c8d0efefcd3",
            "source": "first-token-detail",
            "tokensPerSec": 1434,
            "ttftMs": 5877
          },
          {
            "line": 538,
            "rx": "2026-10-31T06:42:48.448Z",
            "sessionId": "e826520b-467f-4c3a-bf28-df83fc5ff11a",
            "source": "first-token-detail",
            "tokensPerSec": 1210,
            "ttftMs": 6955
          },
          {
            "line": 717,
            "rx": "2026-10-31T06:43:43.397Z",
            "sessionId": "628775e7-0bb9-4f7c-ad15-2c598e322f15",
            "source": "first-token-detail",
            "tokensPerSec": 1281,
            "ttftMs": 6883
          },
          {
            "line": 2296,
            "rx": "2026-10-31T06:48:10.929Z",
            "sessionId": "628775e7-0bb9-4f7c-ad15-2c598e322f15",
            "source": "first-token-detail",
            "tokensPerSec": 337,
            "ttftMs": 465
          },
          {
            "line": 3015,
            "rx": "2026-10-31T06:50:05.496Z",
            "sessionId": "628775e7-0bb9-4f7c-ad15-2c598e322f15",
            "source": "first-token-detail",
            "tokensPerSec": 400,
            "ttftMs": 343
          },
          {
            "line": 3347,
            "rx": "2026-10-31T06:51:13.492Z",
            "sessionId": "db710748-67d9-4089-a45c-4ca83547d1a2",
            "source": "first-token-detail",
            "tokensPerSec": 1251,
            "ttftMs": 6809
          },
          {
            "line": 3651,
            "rx": "2026-10-31T06:52:36.412Z",
            "sessionId": "628775e7-0bb9-4f7c-ad15-2c598e322f15",
            "source": "first-token-detail",
            "tokensPerSec": 998,
            "ttftMs": 16722
          },
          {
            "line": 3709,
            "rx": "2026-10-31T06:52:50.772Z",
            "sessionId": "03c246e1-ea21-4ade-bc66-b91b09e60cfc",
            "source": "first-token-detail",
            "tokensPerSec": 261,
            "ttftMs": 34014
          },
          {
            "line": 3803,
            "rx": "2026-10-31T06:53:37.345Z",
            "sessionId": "628775e7-0bb9-4f7c-ad15-2c598e322f15",
            "source": "first-token-detail",
            "tokensPerSec": 971,
            "ttftMs": 17818
          },
          {
            "line": 3890,
            "rx": "2026-10-31T06:54:24.730Z",
            "sessionId": "628775e7-0bb9-4f7c-ad15-2c598e322f15",
            "source": "first-token-detail",
            "tokensPerSec": 932,
            "ttftMs": 20006
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-invoice-run",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-invoice-run",
        "repetition": 2
      },
      "trialId": "craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-31T07-00-15-530Z-uj35",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-31T07-00-15-530Z-uj35/recording/chat-events.jsonl",
      "recordingSha256": "1246f451cb0dfbad4a27d3ac551ade363e63b57ea885d4911b6eb0bb30b8b89a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-10-31T07-00-15-530Z-uj35/metrics.json",
      "metricsSha256": "f7b379d02f5d500c6184789da2ef1242458d7ee3e7f78f57cca53f37e8ad826f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 532,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-10-31T07:00:43.155Z",
            "sessionId": "859e0439-94fb-4291-a2d2-3aa03d67f7b9",
            "source": "first-token-detail",
            "tokensPerSec": 569,
            "ttftMs": 21112
          },
          {
            "line": 422,
            "rx": "2026-10-31T07:01:59.352Z",
            "sessionId": "53038a51-053f-4dce-aec8-65dcaf06b25d",
            "source": "first-token-detail",
            "tokensPerSec": 632,
            "ttftMs": 12212
          },
          {
            "line": 765,
            "rx": "2026-10-31T07:03:10.007Z",
            "sessionId": "81decaeb-170f-4098-b4e3-3a4d31382bc7",
            "source": "first-token-detail",
            "tokensPerSec": 603,
            "ttftMs": 12896
          },
          {
            "line": 1126,
            "rx": "2026-10-31T07:04:31.055Z",
            "sessionId": "6230b06b-2de8-42c0-9f10-8204e22ae6e0",
            "source": "first-token-detail",
            "tokensPerSec": 556,
            "ttftMs": 13950
          },
          {
            "line": 1514,
            "rx": "2026-10-31T07:06:03.520Z",
            "sessionId": "fc05dad8-c137-47d8-ab88-a7856fe88f21",
            "source": "first-token-detail",
            "tokensPerSec": 513,
            "ttftMs": 16373
          },
          {
            "line": 1920,
            "rx": "2026-10-31T07:07:47.926Z",
            "sessionId": "c72f676b-4e51-464d-bfaa-6ebfab8290b9",
            "source": "first-token-detail",
            "tokensPerSec": 551,
            "ttftMs": 15720
          },
          {
            "line": 3039,
            "rx": "2026-10-31T07:11:55.149Z",
            "sessionId": "fc05dad8-c137-47d8-ab88-a7856fe88f21",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 35571
          },
          {
            "line": 3193,
            "rx": "2026-10-31T07:13:11.995Z",
            "sessionId": "fc05dad8-c137-47d8-ab88-a7856fe88f21",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 32506
          },
          {
            "line": 3200,
            "rx": "2026-10-31T07:13:12.971Z",
            "sessionId": "9a8d095f-f73e-493f-9b16-189d22828eea",
            "source": "first-token-detail",
            "tokensPerSec": 72,
            "ttftMs": 115739
          },
          {
            "line": 3374,
            "rx": "2026-10-31T07:14:36.702Z",
            "sessionId": "fc05dad8-c137-47d8-ab88-a7856fe88f21",
            "source": "first-token-detail",
            "tokensPerSec": 492,
            "ttftMs": 31790
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-author-linear",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-linear",
        "repetition": 2
      },
      "trialId": "craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-31T07-16-05-763Z-q5fx",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-31T07-16-05-763Z-q5fx/recording/chat-events.jsonl",
      "recordingSha256": "e89f33e3e8a55c9011da49621af23c81df393990e91ef9e3ae615dde03316b6c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-gemma4-12b-q4-2026-10-31T07-16-05-763Z-q5fx/metrics.json",
      "metricsSha256": "17184bd70857e23ab69526b572464c7adb1b7570a9e43429a8277763fc9b48f2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 577.5,
        "sampleCount": 16,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 16,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T07:16:26.684Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 1068,
            "ttftMs": 19249
          },
          {
            "line": 411,
            "rx": "2026-10-31T07:18:00.652Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 427
          },
          {
            "line": 538,
            "rx": "2026-10-31T07:18:19.482Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 383,
            "ttftMs": 498
          },
          {
            "line": 692,
            "rx": "2026-10-31T07:18:46.944Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 440,
            "ttftMs": 529
          },
          {
            "line": 971,
            "rx": "2026-10-31T07:19:31.220Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 367,
            "ttftMs": 496
          },
          {
            "line": 1668,
            "rx": "2026-10-31T07:21:58.255Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 557,
            "ttftMs": 985
          },
          {
            "line": 2872,
            "rx": "2026-10-31T07:26:01.364Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 597,
            "ttftMs": 2135
          },
          {
            "line": 3254,
            "rx": "2026-10-31T07:27:02.809Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 559,
            "ttftMs": 1098
          },
          {
            "line": 3761,
            "rx": "2026-10-31T07:28:27.733Z",
            "sessionId": "1ae6313e-68a0-4e3c-a195-3ea8c597c0c2",
            "source": "first-token-detail",
            "tokensPerSec": 759,
            "ttftMs": 554
          },
          {
            "line": 4098,
            "rx": "2026-10-31T07:32:23.929Z",
            "sessionId": "75b99a9f-8f2f-48ad-9b26-4470cd920248",
            "source": "first-token-detail",
            "tokensPerSec": 1849,
            "ttftMs": 7243
          },
          {
            "line": 4194,
            "rx": "2026-10-31T07:32:40.886Z",
            "sessionId": "75b99a9f-8f2f-48ad-9b26-4470cd920248",
            "source": "first-token-detail",
            "tokensPerSec": 984,
            "ttftMs": 1215
          },
          {
            "line": 4289,
            "rx": "2026-10-31T07:32:54.252Z",
            "sessionId": "a603f86c-6a29-42da-b18e-4d29f0e9570f",
            "source": "first-token-detail",
            "tokensPerSec": 529,
            "ttftMs": 18709
          },
          {
            "line": 4590,
            "rx": "2026-10-31T07:34:08.614Z",
            "sessionId": "75b99a9f-8f2f-48ad-9b26-4470cd920248",
            "source": "first-token-detail",
            "tokensPerSec": 543,
            "ttftMs": 476
          },
          {
            "line": 4783,
            "rx": "2026-10-31T07:34:34.146Z",
            "sessionId": "75b99a9f-8f2f-48ad-9b26-4470cd920248",
            "source": "first-token-detail",
            "tokensPerSec": 596,
            "ttftMs": 530
          },
          {
            "line": 5070,
            "rx": "2026-10-31T07:35:17.130Z",
            "sessionId": "a603f86c-6a29-42da-b18e-4d29f0e9570f",
            "source": "first-token-detail",
            "tokensPerSec": 895,
            "ttftMs": 452
          },
          {
            "line": 5101,
            "rx": "2026-10-31T07:43:31.774Z",
            "sessionId": "75b99a9f-8f2f-48ad-9b26-4470cd920248",
            "source": "first-token-detail",
            "tokensPerSec": 1046,
            "ttftMs": 6621
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-author-linear",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-linear",
        "repetition": 2
      },
      "trialId": "craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-31T07-52-52-532Z-liak",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-31T07-52-52-532Z-liak/recording/chat-events.jsonl",
      "recordingSha256": "0f8567e2452bb5b8f2885efe7711c1272470df4d8a5b4301e088809ec85e63b6",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-10-31T07-52-52-532Z-liak/metrics.json",
      "metricsSha256": "44084c5e453e7c1e304f330582f270cee2dc98b32fc0d76150b7e3053e162d6d",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 479.5,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 40,
            "rx": "2026-10-31T07:53:34.851Z",
            "sessionId": "7b60d7b6-7e06-4d0e-8a44-be402ac68a0e",
            "source": "first-token-detail",
            "tokensPerSec": 529,
            "ttftMs": 40751
          },
          {
            "line": 2198,
            "rx": "2026-10-31T08:00:34.148Z",
            "sessionId": "8c6fe5e0-b5f1-4e31-bee4-eb7debfd9e0c",
            "source": "first-token-detail",
            "tokensPerSec": 474,
            "ttftMs": 29800
          },
          {
            "line": 2215,
            "rx": "2026-10-31T08:00:37.429Z",
            "sessionId": "7b60d7b6-7e06-4d0e-8a44-be402ac68a0e",
            "source": "first-token-detail",
            "tokensPerSec": 259,
            "ttftMs": 21911
          },
          {
            "line": 3768,
            "rx": "2026-10-31T08:05:15.570Z",
            "sessionId": "986f85d7-a251-4609-a82b-adb70a8bee3f",
            "source": "first-token-detail",
            "tokensPerSec": 515,
            "ttftMs": 47981
          },
          {
            "line": 6232,
            "rx": "2026-10-31T08:12:21.713Z",
            "sessionId": "403039df-46f6-4996-a0a6-7563a21d5783",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 192232
          },
          {
            "line": 8556,
            "rx": "2026-10-31T08:18:39.754Z",
            "sessionId": "7b60d7b6-7e06-4d0e-8a44-be402ac68a0e",
            "source": "first-token-detail",
            "tokensPerSec": 252,
            "ttftMs": 19307
          },
          {
            "line": 9458,
            "rx": "2026-10-31T08:21:30.661Z",
            "sessionId": "64f0aae2-f9c5-497a-97d0-a7c6bbae7a08",
            "source": "first-token-detail",
            "tokensPerSec": 499,
            "ttftMs": 70919
          },
          {
            "line": 9487,
            "rx": "2026-10-31T08:21:40.622Z",
            "sessionId": "12313543-d035-468a-b166-e88f242c7841",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 27755
          },
          {
            "line": 10076,
            "rx": "2026-10-31T08:23:22.088Z",
            "sessionId": "986f85d7-a251-4609-a82b-adb70a8bee3f",
            "source": "first-token-detail",
            "tokensPerSec": 317,
            "ttftMs": 10471
          },
          {
            "line": 10670,
            "rx": "2026-10-31T08:24:57.326Z",
            "sessionId": "7b60d7b6-7e06-4d0e-8a44-be402ac68a0e",
            "source": "first-token-detail",
            "tokensPerSec": 252,
            "ttftMs": 13372
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-route-multi",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-route-multi",
        "repetition": 2
      },
      "trialId": "craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-31T09-38-41-634Z-awya",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-31T09-38-41-634Z-awya/recording/chat-events.jsonl",
      "recordingSha256": "73b813b5ee1f47df61d9cf45eb078bb25b8f4a93b8ea9aee75eac82694242a64",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-10-31T09-38-41-634Z-awya/metrics.json",
      "metricsSha256": "8b60971e8b749a3fcd1f668c3f4c0afb6964d46325549926e34ac54be5a01054",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 374,
        "sampleCount": 16,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 16,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-10-31T09:39:04.227Z",
            "sessionId": "2976d921-919d-47a4-a309-8927f17a07f6",
            "source": "first-token-detail",
            "tokensPerSec": 827,
            "ttftMs": 20891
          },
          {
            "line": 331,
            "rx": "2026-10-31T09:40:01.195Z",
            "sessionId": "2976d921-919d-47a4-a309-8927f17a07f6",
            "source": "first-token-detail",
            "tokensPerSec": 387,
            "ttftMs": 894
          },
          {
            "line": 551,
            "rx": "2026-10-31T09:40:49.401Z",
            "sessionId": "2976d921-919d-47a4-a309-8927f17a07f6",
            "source": "first-token-detail",
            "tokensPerSec": 361,
            "ttftMs": 6934
          },
          {
            "line": 699,
            "rx": "2026-10-31T09:41:16.817Z",
            "sessionId": "aa0cc8f1-0fad-4092-8fd0-e2ea070e2a21",
            "source": "first-token-detail",
            "tokensPerSec": 240,
            "ttftMs": 48483
          },
          {
            "line": 736,
            "rx": "2026-10-31T09:41:46.030Z",
            "sessionId": "876cf4ee-8b65-4619-bb81-88a0d7697afb",
            "source": "first-token-detail",
            "tokensPerSec": 600,
            "ttftMs": 36522
          },
          {
            "line": 1349,
            "rx": "2026-10-31T09:43:51.085Z",
            "sessionId": "79ff9bf8-28c4-4844-af51-94c1ab2796dd",
            "source": "first-token-detail",
            "tokensPerSec": 435,
            "ttftMs": 27766
          },
          {
            "line": 1547,
            "rx": "2026-10-31T09:44:47.472Z",
            "sessionId": "0ee15a4a-5b33-405c-8be4-66c41d28a688",
            "source": "first-token-detail",
            "tokensPerSec": 451,
            "ttftMs": 30007
          },
          {
            "line": 1916,
            "rx": "2026-10-31T09:46:01.379Z",
            "sessionId": "0ee15a4a-5b33-405c-8be4-66c41d28a688",
            "source": "first-token-detail",
            "tokensPerSec": 337,
            "ttftMs": 3412
          },
          {
            "line": 2852,
            "rx": "2026-10-31T09:48:57.948Z",
            "sessionId": "0ee15a4a-5b33-405c-8be4-66c41d28a688",
            "source": "first-token-detail",
            "tokensPerSec": 217,
            "ttftMs": 1653
          },
          {
            "line": 4294,
            "rx": "2026-10-31T09:53:18.948Z",
            "sessionId": "79ff9bf8-28c4-4844-af51-94c1ab2796dd",
            "source": "first-token-detail",
            "tokensPerSec": 283,
            "ttftMs": 34942
          },
          {
            "line": 4530,
            "rx": "2026-10-31T09:54:21.321Z",
            "sessionId": "0ee15a4a-5b33-405c-8be4-66c41d28a688",
            "source": "first-token-detail",
            "tokensPerSec": 492,
            "ttftMs": 37758
          },
          {
            "line": 4763,
            "rx": "2026-10-31T09:55:26.482Z",
            "sessionId": "2976d921-919d-47a4-a309-8927f17a07f6",
            "source": "first-token-detail",
            "tokensPerSec": 305,
            "ttftMs": 19611
          },
          {
            "line": 5280,
            "rx": "2026-10-31T09:57:00.633Z",
            "sessionId": "0809d79d-71ce-4695-a348-f84610fff672",
            "source": "first-token-detail",
            "tokensPerSec": 503,
            "ttftMs": 46947
          },
          {
            "line": 5335,
            "rx": "2026-10-31T09:57:09.352Z",
            "sessionId": "876cf4ee-8b65-4619-bb81-88a0d7697afb",
            "source": "first-token-detail",
            "tokensPerSec": 353,
            "ttftMs": 26037
          },
          {
            "line": 6971,
            "rx": "2026-10-31T10:01:37.095Z",
            "sessionId": "2976d921-919d-47a4-a309-8927f17a07f6",
            "source": "first-token-detail",
            "tokensPerSec": 233,
            "ttftMs": 144048
          },
          {
            "line": 8451,
            "rx": "2026-10-31T10:06:11.097Z",
            "sessionId": "119acb12-e4d9-4091-8bcc-6bffbf98cbef",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 62357
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-route-multi",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-route-multi",
        "repetition": 2
      },
      "trialId": "craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-31T10-31-57-276Z-cukg",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-31T10-31-57-276Z-cukg/recording/chat-events.jsonl",
      "recordingSha256": "8ec00b9f2cbe85de51c51e71eb990d97c5b4d7acfc8f5d7e70e7ca44a932af6c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-gemma4-12b-q4-2026-10-31T10-31-57-276Z-cukg/metrics.json",
      "metricsSha256": "4036c038ccd555a5d9d60a2ad234959e35b87e24941499a1b54b686762c4b300",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1015,
        "sampleCount": 23,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 23,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-10-31T10:32:14.637Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 1085,
            "ttftMs": 15638
          },
          {
            "line": 1470,
            "rx": "2026-10-31T10:36:22.633Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 271,
            "ttftMs": 412
          },
          {
            "line": 3064,
            "rx": "2026-10-31T10:40:57.001Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 240,
            "ttftMs": 461
          },
          {
            "line": 3261,
            "rx": "2026-10-31T10:41:28.217Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 339,
            "ttftMs": 441
          },
          {
            "line": 4890,
            "rx": "2026-10-31T10:45:59.388Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 293,
            "ttftMs": 1546
          },
          {
            "line": 5021,
            "rx": "2026-10-31T10:46:23.965Z",
            "sessionId": "b965b070-27c5-4f2b-a6f4-c2a5208ef67e",
            "source": "first-token-detail",
            "tokensPerSec": 315,
            "ttftMs": 234623
          },
          {
            "line": 5098,
            "rx": "2026-10-31T10:46:38.818Z",
            "sessionId": "45e37638-99d9-4b9c-8e52-91e753a6a067",
            "source": "first-token-detail",
            "tokensPerSec": 984,
            "ttftMs": 17575
          },
          {
            "line": 5266,
            "rx": "2026-10-31T10:47:25.349Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 595,
            "ttftMs": 787
          },
          {
            "line": 5747,
            "rx": "2026-10-31T10:48:43.827Z",
            "sessionId": "39ac80e5-9a6d-4597-9998-d4cda705d74c",
            "source": "first-token-detail",
            "tokensPerSec": 1028,
            "ttftMs": 39540
          },
          {
            "line": 5863,
            "rx": "2026-10-31T10:49:05.063Z",
            "sessionId": "45e37638-99d9-4b9c-8e52-91e753a6a067",
            "source": "first-token-detail",
            "tokensPerSec": 1044,
            "ttftMs": 29452
          },
          {
            "line": 7736,
            "rx": "2026-10-31T10:54:11.732Z",
            "sessionId": "a03a76e8-6da2-4cbb-90c2-01b5d633ecee",
            "source": "first-token-detail",
            "tokensPerSec": 1036,
            "ttftMs": 244871
          },
          {
            "line": 7880,
            "rx": "2026-10-31T10:54:40.812Z",
            "sessionId": "45e37638-99d9-4b9c-8e52-91e753a6a067",
            "source": "first-token-detail",
            "tokensPerSec": 1017,
            "ttftMs": 10477
          },
          {
            "line": 8502,
            "rx": "2026-10-31T10:56:29.013Z",
            "sessionId": "39ac80e5-9a6d-4597-9998-d4cda705d74c",
            "source": "first-token-detail",
            "tokensPerSec": 1015,
            "ttftMs": 84500
          },
          {
            "line": 8954,
            "rx": "2026-10-31T10:57:40.403Z",
            "sessionId": "40ecbb9c-b283-40f6-a7d5-7e029fb650ad",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 6219
          },
          {
            "line": 8990,
            "rx": "2026-10-31T10:57:48.188Z",
            "sessionId": "39ac80e5-9a6d-4597-9998-d4cda705d74c",
            "source": "first-token-detail",
            "tokensPerSec": 564,
            "ttftMs": 6147
          },
          {
            "line": 9153,
            "rx": "2026-10-31T10:58:20.056Z",
            "sessionId": "fdb9256e-54b7-4e07-ae3e-4cc3ab32f6e2",
            "source": "first-token-detail",
            "tokensPerSec": 614,
            "ttftMs": 734
          },
          {
            "line": 9262,
            "rx": "2026-10-31T10:58:44.148Z",
            "sessionId": "40ecbb9c-b283-40f6-a7d5-7e029fb650ad",
            "source": "first-token-detail",
            "tokensPerSec": 968,
            "ttftMs": 6525
          },
          {
            "line": 9437,
            "rx": "2026-10-31T10:59:16.466Z",
            "sessionId": "b7765d97-358c-4589-81ec-b5131422e495",
            "source": "first-token-detail",
            "tokensPerSec": 1073,
            "ttftMs": 37007
          },
          {
            "line": 9503,
            "rx": "2026-10-31T10:59:31.212Z",
            "sessionId": "40ecbb9c-b283-40f6-a7d5-7e029fb650ad",
            "source": "first-token-detail",
            "tokensPerSec": 1064,
            "ttftMs": 23517
          },
          {
            "line": 9604,
            "rx": "2026-10-31T11:00:04.859Z",
            "sessionId": "40ecbb9c-b283-40f6-a7d5-7e029fb650ad",
            "source": "first-token-detail",
            "tokensPerSec": 1040,
            "ttftMs": 8774
          },
          {
            "line": 9733,
            "rx": "2026-10-31T11:00:59.798Z",
            "sessionId": "b7765d97-358c-4589-81ec-b5131422e495",
            "source": "first-token-detail",
            "tokensPerSec": 1096,
            "ttftMs": 10320
          },
          {
            "line": 10052,
            "rx": "2026-10-31T11:01:55.818Z",
            "sessionId": "8c8b5766-a04b-4652-80f7-ae42c5cf0e84",
            "source": "first-token-detail",
            "tokensPerSec": 1123,
            "ttftMs": 4745
          },
          {
            "line": 10111,
            "rx": "2026-10-31T11:02:21.796Z",
            "sessionId": "f38b70ea-f27f-410e-b011-d93b9d6cc0a0",
            "source": "first-token-detail",
            "tokensPerSec": 1071,
            "ttftMs": 7437
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-export-generalize",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-export-generalize",
        "repetition": 2
      },
      "trialId": "craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-31T11-03-10-332Z-nahg",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-31T11-03-10-332Z-nahg/recording/chat-events.jsonl",
      "recordingSha256": "32d30e3021f61f975e66a4b2b41e72c74fc2711a48d01e6afe53cd6fc4db18be",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-10-31T11-03-10-332Z-nahg/metrics.json",
      "metricsSha256": "1315fcc4e50ccb9c0d939a1d0401893da189ba7e6e784b764c47e8cd66dd8ec1",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 503,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 36,
            "rx": "2026-10-31T11:03:51.222Z",
            "sessionId": "55ef07f5-6322-4dc1-8c58-789a49019e5a",
            "source": "first-token-detail",
            "tokensPerSec": 553,
            "ttftMs": 39220
          },
          {
            "line": 6556,
            "rx": "2026-10-31T11:23:01.115Z",
            "sessionId": "e4f67232-34a7-4205-bdad-4c04f14f6acd",
            "source": "first-token-detail",
            "tokensPerSec": 513,
            "ttftMs": 9400
          },
          {
            "line": 7188,
            "rx": "2026-10-31T11:24:49.320Z",
            "sessionId": "55ef07f5-6322-4dc1-8c58-789a49019e5a",
            "source": "first-token-detail",
            "tokensPerSec": 298,
            "ttftMs": 2192
          },
          {
            "line": 8704,
            "rx": "2026-10-31T11:30:37.938Z",
            "sessionId": "e4f67232-34a7-4205-bdad-4c04f14f6acd",
            "source": "first-token-detail",
            "tokensPerSec": 503,
            "ttftMs": 11128
          },
          {
            "line": 8732,
            "rx": "2026-10-31T11:30:54.824Z",
            "sessionId": "ad7d500c-6565-4784-ab0c-3524cf1d7e30",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 26752
          },
          {
            "line": 9265,
            "rx": "2026-10-31T11:32:26.466Z",
            "sessionId": "55ef07f5-6322-4dc1-8c58-789a49019e5a",
            "source": "first-token-detail",
            "tokensPerSec": 272,
            "ttftMs": 76124
          },
          {
            "line": 11895,
            "rx": "2026-10-31T11:41:43.310Z",
            "sessionId": "e4f67232-34a7-4205-bdad-4c04f14f6acd",
            "source": "first-token-detail",
            "tokensPerSec": 486,
            "ttftMs": 31775
          },
          {
            "line": 12131,
            "rx": "2026-10-31T11:42:45.149Z",
            "sessionId": "a6ec5590-03d4-4e48-a062-dad6a60f2768",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 21354
          },
          {
            "line": 13799,
            "rx": "2026-10-31T11:47:55.815Z",
            "sessionId": "e4f67232-34a7-4205-bdad-4c04f14f6acd",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 12305
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-export-generalize",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-export-generalize",
        "repetition": 2
      },
      "trialId": "craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-31T12-57-22-612Z-hijn",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-31T12-57-22-612Z-hijn/recording/chat-events.jsonl",
      "recordingSha256": "0ec95a9b45b16a60a1a15190df4139175a46853c1d76ca179cc39bfaeb81abea",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-gemma4-12b-q4-2026-10-31T12-57-22-612Z-hijn/metrics.json",
      "metricsSha256": "a584b2e525eb732cd7de638a349e648d5710e08fa2d3b51c24a0fafe4b7ebbe5",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 851,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-10-31T12:57:45.822Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 957,
            "ttftMs": 21544
          },
          {
            "line": 245,
            "rx": "2026-10-31T12:58:46.642Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 542,
            "ttftMs": 636
          },
          {
            "line": 366,
            "rx": "2026-10-31T12:59:05.545Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 827,
            "ttftMs": 788
          },
          {
            "line": 1267,
            "rx": "2026-10-31T13:01:43.557Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 452,
            "ttftMs": 617
          },
          {
            "line": 1417,
            "rx": "2026-10-31T13:02:07.758Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 851,
            "ttftMs": 1090
          },
          {
            "line": 1895,
            "rx": "2026-10-31T13:03:29.374Z",
            "sessionId": "d0bea949-cbae-49c8-bbc5-693f78865a3c",
            "source": "first-token-detail",
            "tokensPerSec": 1052,
            "ttftMs": 3492
          },
          {
            "line": 1924,
            "rx": "2026-10-31T13:03:36.152Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 683
          },
          {
            "line": 2568,
            "rx": "2026-10-31T13:05:27.306Z",
            "sessionId": "d0bea949-cbae-49c8-bbc5-693f78865a3c",
            "source": "first-token-detail",
            "tokensPerSec": 966,
            "ttftMs": 5767
          },
          {
            "line": 2592,
            "rx": "2026-10-31T13:05:32.973Z",
            "sessionId": "00591a96-dee2-4ab4-8f62-a82b522efbad",
            "source": "first-token-detail",
            "tokensPerSec": 521,
            "ttftMs": 763
          },
          {
            "line": 2791,
            "rx": "2026-10-31T13:06:03.388Z",
            "sessionId": "d0bea949-cbae-49c8-bbc5-693f78865a3c",
            "source": "first-token-detail",
            "tokensPerSec": 636,
            "ttftMs": 1041
          },
          {
            "line": 2873,
            "rx": "2026-10-31T13:06:39.940Z",
            "sessionId": "d0bea949-cbae-49c8-bbc5-693f78865a3c",
            "source": "first-token-detail",
            "tokensPerSec": 1169,
            "ttftMs": 5796
          },
          {
            "line": 2908,
            "rx": "2026-10-31T13:06:49.982Z",
            "sessionId": "d0bea949-cbae-49c8-bbc5-693f78865a3c",
            "source": "first-token-detail",
            "tokensPerSec": 1202,
            "ttftMs": 712
          },
          {
            "line": 2940,
            "rx": "2026-10-31T13:07:08.567Z",
            "sessionId": "d0bea949-cbae-49c8-bbc5-693f78865a3c",
            "source": "first-token-detail",
            "tokensPerSec": 947,
            "ttftMs": 9202
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-author-params",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-params",
        "repetition": 2
      },
      "trialId": "craftbook-author-params-mlx-gemma4-12b-q4-2026-10-31T13-38-43-140Z-yyk8",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-gemma4-12b-q4-2026-10-31T13-38-43-140Z-yyk8/recording/chat-events.jsonl",
      "recordingSha256": "7c2b67f8c74b838ef3c45438f518dea238a475a75bfd6519153626a4dd01a34b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-gemma4-12b-q4-2026-10-31T13-38-43-140Z-yyk8/metrics.json",
      "metricsSha256": "98d2e3a38adbff2a3ef328d48d76f114db90620851fe1eac9349b578acad6e3f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 698,
        "sampleCount": 13,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 13,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T13:39:04.164Z",
            "sessionId": "0899ae59-229a-463e-9b24-414a4b23125c",
            "source": "first-token-detail",
            "tokensPerSec": 1062,
            "ttftMs": 19335
          },
          {
            "line": 321,
            "rx": "2026-10-31T13:40:13.097Z",
            "sessionId": "0899ae59-229a-463e-9b24-414a4b23125c",
            "source": "first-token-detail",
            "tokensPerSec": 580,
            "ttftMs": 700
          },
          {
            "line": 597,
            "rx": "2026-10-31T13:41:03.598Z",
            "sessionId": "0899ae59-229a-463e-9b24-414a4b23125c",
            "source": "first-token-detail",
            "tokensPerSec": 305,
            "ttftMs": 397
          },
          {
            "line": 651,
            "rx": "2026-10-31T13:41:16.001Z",
            "sessionId": "0899ae59-229a-463e-9b24-414a4b23125c",
            "source": "first-token-detail",
            "tokensPerSec": 810,
            "ttftMs": 537
          },
          {
            "line": 987,
            "rx": "2026-10-31T13:42:15.299Z",
            "sessionId": "0899ae59-229a-463e-9b24-414a4b23125c",
            "source": "first-token-detail",
            "tokensPerSec": 307,
            "ttftMs": 401
          },
          {
            "line": 1107,
            "rx": "2026-10-31T13:49:27.602Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 1847,
            "ttftMs": 12880
          },
          {
            "line": 1217,
            "rx": "2026-10-31T13:49:46.054Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 989,
            "ttftMs": 555
          },
          {
            "line": 1319,
            "rx": "2026-10-31T13:50:00.149Z",
            "sessionId": "ac2aca6c-7998-45b1-8a9c-c389d8339d65",
            "source": "first-token-detail",
            "tokensPerSec": 510,
            "ttftMs": 19323
          },
          {
            "line": 1433,
            "rx": "2026-10-31T13:50:09.902Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 698,
            "ttftMs": 376
          },
          {
            "line": 1585,
            "rx": "2026-10-31T13:50:29.921Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 781,
            "ttftMs": 657
          },
          {
            "line": 1760,
            "rx": "2026-10-31T13:50:56.891Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 578,
            "ttftMs": 475
          },
          {
            "line": 1847,
            "rx": "2026-10-31T13:51:07.252Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 488,
            "ttftMs": 435
          },
          {
            "line": 2025,
            "rx": "2026-10-31T13:51:58.785Z",
            "sessionId": "2b7fcd7e-ad2b-462e-9acd-931d45dc7b8b",
            "source": "first-token-detail",
            "tokensPerSec": 1106,
            "ttftMs": 12228
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-author-params",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-params",
        "repetition": 2
      },
      "trialId": "craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-31T15-10-08-307Z-rq1v",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-31T15-10-08-307Z-rq1v/recording/chat-events.jsonl",
      "recordingSha256": "bdada36d590020d636cee2d5f4b3ac26fd3407540ca612e4c1f19b07b9e152bd",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-qwen3-8-27b-q4-2026-10-31T15-10-08-307Z-rq1v/metrics.json",
      "metricsSha256": "4c45fa03af6cbc7a712a683591adc8e7f127c83559ab30baeb12cfb859707fdc",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 301,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 35,
            "rx": "2026-10-31T15:10:39.707Z",
            "sessionId": "5ab67624-5185-4d97-a1c9-dc5833469109",
            "source": "first-token-detail",
            "tokensPerSec": 762,
            "ttftMs": 29721
          },
          {
            "line": 1971,
            "rx": "2026-10-31T15:16:57.840Z",
            "sessionId": "5ab67624-5185-4d97-a1c9-dc5833469109",
            "source": "first-token-detail",
            "tokensPerSec": 302,
            "ttftMs": 9278
          },
          {
            "line": 2089,
            "rx": "2026-10-31T15:17:21.827Z",
            "sessionId": "ff96f7eb-705c-4620-8cf0-5aadddc14b57",
            "source": "first-token-detail",
            "tokensPerSec": 201,
            "ttftMs": 36739
          },
          {
            "line": 2355,
            "rx": "2026-10-31T15:18:19.460Z",
            "sessionId": "5ab67624-5185-4d97-a1c9-dc5833469109",
            "source": "first-token-detail",
            "tokensPerSec": 223,
            "ttftMs": 2505
          },
          {
            "line": 2694,
            "rx": "2026-10-31T15:19:35.228Z",
            "sessionId": "5ab67624-5185-4d97-a1c9-dc5833469109",
            "source": "first-token-detail",
            "tokensPerSec": 300,
            "ttftMs": 4278
          },
          {
            "line": 4372,
            "rx": "2026-10-31T15:24:18.714Z",
            "sessionId": "5ab67624-5185-4d97-a1c9-dc5833469109",
            "source": "first-token-detail",
            "tokensPerSec": 252,
            "ttftMs": 9730
          },
          {
            "line": 5691,
            "rx": "2026-10-31T15:28:00.665Z",
            "sessionId": "0dca5dbe-2f9f-4bae-8e1b-04b90e0f9c78",
            "source": "first-token-detail",
            "tokensPerSec": 493,
            "ttftMs": 60383
          },
          {
            "line": 6309,
            "rx": "2026-10-31T15:29:41.181Z",
            "sessionId": "5ab67624-5185-4d97-a1c9-dc5833469109",
            "source": "first-token-detail",
            "tokensPerSec": 246,
            "ttftMs": 46163
          },
          {
            "line": 6396,
            "rx": "2026-10-31T15:30:05.417Z",
            "sessionId": "36f37022-da31-40cd-b19f-9ff823343817",
            "source": "first-token-detail",
            "tokensPerSec": 490,
            "ttftMs": 25053
          },
          {
            "line": 7264,
            "rx": "2026-10-31T15:32:38.665Z",
            "sessionId": "ff96f7eb-705c-4620-8cf0-5aadddc14b57",
            "source": "first-token-detail",
            "tokensPerSec": 483,
            "ttftMs": 88192
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-author-gate-script",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-gate-script",
        "repetition": 2
      },
      "trialId": "craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-31T15-40-39-897Z-jm3h",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-31T15-40-39-897Z-jm3h/recording/chat-events.jsonl",
      "recordingSha256": "abd38cbbbd386cd4962586c5e1c5369cddb642d6d58b8fdc7d80b72c5d56b020",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-10-31T15-40-39-897Z-jm3h/metrics.json",
      "metricsSha256": "b5178b6bfd8759fe226246ec68e0d9101f46ef358c32d554bc315b926eb7cc2f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 770,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T15:41:00.929Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 1059,
            "ttftMs": 19367
          },
          {
            "line": 489,
            "rx": "2026-10-31T15:42:43.225Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 578,
            "ttftMs": 864
          },
          {
            "line": 1069,
            "rx": "2026-10-31T15:44:51.126Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 445,
            "ttftMs": 703
          },
          {
            "line": 1271,
            "rx": "2026-10-31T15:45:23.244Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 466,
            "ttftMs": 629
          },
          {
            "line": 1584,
            "rx": "2026-10-31T15:46:23.822Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 612
          },
          {
            "line": 1831,
            "rx": "2026-10-31T15:48:24.672Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 770,
            "ttftMs": 467
          },
          {
            "line": 2129,
            "rx": "2026-10-31T15:49:09.978Z",
            "sessionId": "6c1d6f94-c0a0-4d8d-bc28-53c065ae1792",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 500
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-author-gate-script",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-gate-script",
        "repetition": 2
      },
      "trialId": "craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-31T16-03-12-666Z-66bx",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-31T16-03-12-666Z-66bx/recording/chat-events.jsonl",
      "recordingSha256": "30677a02eeb08449a7b04fd7c7ec47722a88a94fb36cd5a21a0622aa234ddcf9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-10-31T16-03-12-666Z-66bx/metrics.json",
      "metricsSha256": "6a57196dd99b3740a27d624a8c2797eddcaf2a328d1d64f3022705eecdb8a1cf",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 533,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 38,
            "rx": "2026-10-31T16:03:54.829Z",
            "sessionId": "8cfe37a2-f185-4d7f-8f3e-9e094b78d9b3",
            "source": "first-token-detail",
            "tokensPerSec": 533,
            "ttftMs": 40494
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-qwen-mlx-complex-work-craftbook-author-fanout",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-fanout",
        "repetition": 2
      },
      "trialId": "craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-31T18-41-32-968Z-wq8e",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-31T18-41-32-968Z-wq8e/recording/chat-events.jsonl",
      "recordingSha256": "eb91a4467847651952ec738b150ad30c3e6897cea1d260bf32cd7a0bf9937931",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-qwen-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-10-31T18-41-32-968Z-wq8e/metrics.json",
      "metricsSha256": "808cbea1c05c3b8b4e9157189fb7eb3f2cee529ca17ac8217e5bdf9643c356a2",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 399,
        "sampleCount": 16,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 16,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 35,
            "rx": "2026-10-31T18:42:04.431Z",
            "sessionId": "962f0277-f9ab-46df-ae49-5a1e054e1408",
            "source": "first-token-detail",
            "tokensPerSec": 774,
            "ttftMs": 29896
          },
          {
            "line": 3418,
            "rx": "2026-10-31T18:53:12.465Z",
            "sessionId": "964b6963-74df-454d-b7e4-af8a9d4195b4",
            "source": "first-token-detail",
            "tokensPerSec": 103,
            "ttftMs": 97644
          },
          {
            "line": 6779,
            "rx": "2026-10-31T19:06:29.986Z",
            "sessionId": "962f0277-f9ab-46df-ae49-5a1e054e1408",
            "source": "first-token-detail",
            "tokensPerSec": 192,
            "ttftMs": 3428
          },
          {
            "line": 9071,
            "rx": "2026-10-31T19:16:12.611Z",
            "sessionId": "962f0277-f9ab-46df-ae49-5a1e054e1408",
            "source": "first-token-detail",
            "tokensPerSec": 205,
            "ttftMs": 2787
          },
          {
            "line": 14954,
            "rx": "2026-10-31T19:34:53.256Z",
            "sessionId": "964b6963-74df-454d-b7e4-af8a9d4195b4",
            "source": "first-token-detail",
            "tokensPerSec": 404,
            "ttftMs": 42479
          },
          {
            "line": 15055,
            "rx": "2026-10-31T19:35:41.939Z",
            "sessionId": "14a8d1a0-8d7b-4862-aa18-1e9614d2bcd2",
            "source": "first-token-detail",
            "tokensPerSec": 424,
            "ttftMs": 60337
          },
          {
            "line": 21438,
            "rx": "2026-10-31T19:54:52.382Z",
            "sessionId": "962f0277-f9ab-46df-ae49-5a1e054e1408",
            "source": "first-token-detail",
            "tokensPerSec": 189,
            "ttftMs": 3074
          },
          {
            "line": 38038,
            "rx": "2026-10-31T20:50:24.865Z",
            "sessionId": "14a8d1a0-8d7b-4862-aa18-1e9614d2bcd2",
            "source": "first-token-detail",
            "tokensPerSec": 263,
            "ttftMs": 3168
          },
          {
            "line": 39458,
            "rx": "2026-10-31T20:54:20.457Z",
            "sessionId": "ab58b009-c661-40c9-85c5-c221c9efe650",
            "source": "first-token-detail",
            "tokensPerSec": 459,
            "ttftMs": 228231
          },
          {
            "line": 41356,
            "rx": "2026-10-31T20:59:53.817Z",
            "sessionId": "1f5915cc-b034-4cf8-95bf-6f01931e2728",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 26316
          },
          {
            "line": 45600,
            "rx": "2026-10-31T21:12:08.934Z",
            "sessionId": "47ae6a52-d989-474a-8af6-44e4488011f0",
            "source": "first-token-detail",
            "tokensPerSec": 462,
            "ttftMs": 261301
          },
          {
            "line": 45911,
            "rx": "2026-10-31T21:12:58.102Z",
            "sessionId": "14a8d1a0-8d7b-4862-aa18-1e9614d2bcd2",
            "source": "first-token-detail",
            "tokensPerSec": 240,
            "ttftMs": 2260
          },
          {
            "line": 50285,
            "rx": "2026-10-31T21:34:31.181Z",
            "sessionId": "962f0277-f9ab-46df-ae49-5a1e054e1408",
            "source": "first-token-detail",
            "tokensPerSec": 283,
            "ttftMs": 1295363
          },
          {
            "line": 50905,
            "rx": "2026-10-31T21:37:33.972Z",
            "sessionId": "14a8d1a0-8d7b-4862-aa18-1e9614d2bcd2",
            "source": "first-token-detail",
            "tokensPerSec": 394,
            "ttftMs": 95243
          },
          {
            "line": 50956,
            "rx": "2026-10-31T21:37:46.782Z",
            "sessionId": "78a97456-66e0-4a50-bde6-a8fe9f1370d5",
            "source": "first-token-detail",
            "tokensPerSec": 458,
            "ttftMs": 103544
          },
          {
            "line": 51556,
            "rx": "2026-10-31T21:39:56.907Z",
            "sessionId": "5cef2dbb-cf41-45e2-9416-a856a8eed347",
            "source": "first-token-detail",
            "tokensPerSec": 497,
            "ttftMs": 21224
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r2-gemma-mlx-complex-work-craftbook-author-fanout",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-fanout",
        "repetition": 2
      },
      "trialId": "craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-31T22-32-10-078Z-dekr",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-31T22-32-10-078Z-dekr/recording/chat-events.jsonl",
      "recordingSha256": "bdc52e4b56b1a7d8913cac6773621b0ad3f8a30efea37172683179628b5967bd",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r2-gemma-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-gemma4-12b-q4-2026-10-31T22-32-10-078Z-dekr/metrics.json",
      "metricsSha256": "ca6873f7a8710bf810074f0e2d4641cee50547e6426c0d13634b5eeaa81b5a4a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 481,
        "sampleCount": 18,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 18,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-10-31T22:32:30.980Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 1071,
            "ttftMs": 19232
          },
          {
            "line": 383,
            "rx": "2026-10-31T22:33:44.914Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 377,
            "ttftMs": 441
          },
          {
            "line": 845,
            "rx": "2026-10-31T22:34:57.103Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 413,
            "ttftMs": 534
          },
          {
            "line": 1072,
            "rx": "2026-10-31T22:35:34.650Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 372,
            "ttftMs": 590
          },
          {
            "line": 1397,
            "rx": "2026-10-31T22:36:25.955Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 447,
            "ttftMs": 642
          },
          {
            "line": 1777,
            "rx": "2026-10-31T22:37:31.527Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 967
          },
          {
            "line": 2077,
            "rx": "2026-10-31T22:38:23.004Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 515,
            "ttftMs": 959
          },
          {
            "line": 2522,
            "rx": "2026-10-31T22:39:37.504Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 389,
            "ttftMs": 764
          },
          {
            "line": 2864,
            "rx": "2026-10-31T22:40:30.946Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 380,
            "ttftMs": 637
          },
          {
            "line": 3003,
            "rx": "2026-10-31T22:40:52.615Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 325,
            "ttftMs": 604
          },
          {
            "line": 3366,
            "rx": "2026-10-31T22:42:01.945Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 1111,
            "ttftMs": 634
          },
          {
            "line": 3865,
            "rx": "2026-10-31T22:43:25.313Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 402,
            "ttftMs": 816
          },
          {
            "line": 4365,
            "rx": "2026-10-31T22:44:49.583Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 318,
            "ttftMs": 605
          },
          {
            "line": 4466,
            "rx": "2026-10-31T22:45:08.491Z",
            "sessionId": "cd0162e0-80db-41df-8ecb-8a780e99585c",
            "source": "first-token-detail",
            "tokensPerSec": 860,
            "ttftMs": 637
          },
          {
            "line": 4848,
            "rx": "2026-10-31T22:49:17.872Z",
            "sessionId": "940c2c74-0a7e-45eb-be73-aef1388fd717",
            "source": "first-token-detail",
            "tokensPerSec": 1820,
            "ttftMs": 7364
          },
          {
            "line": 4945,
            "rx": "2026-10-31T22:49:32.134Z",
            "sessionId": "940c2c74-0a7e-45eb-be73-aef1388fd717",
            "source": "first-token-detail",
            "tokensPerSec": 1028,
            "ttftMs": 1434
          },
          {
            "line": 4995,
            "rx": "2026-10-31T22:49:40.573Z",
            "sessionId": "47b0159a-31f5-4b12-9c55-9f9591292fdf",
            "source": "first-token-detail",
            "tokensPerSec": 894,
            "ttftMs": 11124
          },
          {
            "line": 5179,
            "rx": "2026-10-31T22:52:38.642Z",
            "sessionId": "940c2c74-0a7e-45eb-be73-aef1388fd717",
            "source": "first-token-detail",
            "tokensPerSec": 1184,
            "ttftMs": 1142
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-tictactoe",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "tictactoe",
        "repetition": 3
      },
      "trialId": "tictactoe-mlx-gemma4-12b-q4-2026-10-31T22-53-30-091Z-127v",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-10-31T22-53-30-091Z-127v/recording/chat-events.jsonl",
      "recordingSha256": "e6f319ad5d0e3327c16878359d5d250a8b7931df87c38e742ab9ccf0bb6c1b4f",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-tictactoe/tictactoe/tictactoe-mlx-gemma4-12b-q4-2026-10-31T22-53-30-091Z-127v/metrics.json",
      "metricsSha256": "4fe470cf7d54c1d5cba60205668f00004861acea37948e17b58b61e699eadcca",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1338,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 22,
            "rx": "2026-10-31T22:53:43.480Z",
            "sessionId": "e5fd9996-9027-4301-812c-48376f6e6cac",
            "source": "first-token-detail",
            "tokensPerSec": 1864,
            "ttftMs": 11714
          },
          {
            "line": 125,
            "rx": "2026-10-31T22:54:05.927Z",
            "sessionId": "602f57be-ec2b-4a59-b83a-6d9b6f0946ea",
            "source": "first-token-detail",
            "tokensPerSec": 1550,
            "ttftMs": 4292
          },
          {
            "line": 148,
            "rx": "2026-10-31T22:54:16.860Z",
            "sessionId": "602f57be-ec2b-4a59-b83a-6d9b6f0946ea",
            "source": "first-token-detail",
            "tokensPerSec": 1338,
            "ttftMs": 7386
          },
          {
            "line": 322,
            "rx": "2026-10-31T22:55:06.787Z",
            "sessionId": "e22a3e3b-a20c-41f3-ab38-87daf161c31f",
            "source": "first-token-detail",
            "tokensPerSec": 1405,
            "ttftMs": 3175
          },
          {
            "line": 652,
            "rx": "2026-10-31T22:56:54.520Z",
            "sessionId": "602f57be-ec2b-4a59-b83a-6d9b6f0946ea",
            "source": "first-token-detail",
            "tokensPerSec": 785,
            "ttftMs": 47804
          },
          {
            "line": 1116,
            "rx": "2026-10-31T22:59:00.484Z",
            "sessionId": "e22a3e3b-a20c-41f3-ab38-87daf161c31f",
            "source": "first-token-detail",
            "tokensPerSec": 902,
            "ttftMs": 12946
          },
          {
            "line": 1651,
            "rx": "2026-10-31T23:01:29.315Z",
            "sessionId": "602f57be-ec2b-4a59-b83a-6d9b6f0946ea",
            "source": "first-token-detail",
            "tokensPerSec": 79,
            "ttftMs": 162397
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-tictactoe",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "tictactoe",
        "repetition": 3
      },
      "trialId": "tictactoe-mlx-qwen3-8-27b-q4-2026-10-31T23-04-26-418Z-z6yq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-10-31T23-04-26-418Z-z6yq/recording/chat-events.jsonl",
      "recordingSha256": "74fa51cc1b35c4839ae293fbf9db24f45571174ab1428b7b3ffdc2941824f367",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-tictactoe/tictactoe/tictactoe-mlx-qwen3-8-27b-q4-2026-10-31T23-04-26-418Z-z6yq/metrics.json",
      "metricsSha256": "d7d5150f17b0c9c808196dbdc3f96e7461fcd8b2a293454a3343bfbb32d41994",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 544.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-10-31T23:04:48.557Z",
            "sessionId": "47df5bd7-f7ea-4936-8e47-10569a37f003",
            "source": "first-token-detail",
            "tokensPerSec": 819,
            "ttftMs": 20458
          },
          {
            "line": 100,
            "rx": "2026-10-31T23:04:58.447Z",
            "sessionId": "47df5bd7-f7ea-4936-8e47-10569a37f003",
            "source": "first-token-detail",
            "tokensPerSec": 318,
            "ttftMs": 599
          },
          {
            "line": 129,
            "rx": "2026-10-31T23:05:01.963Z",
            "sessionId": "47df5bd7-f7ea-4936-8e47-10569a37f003",
            "source": "first-token-detail",
            "tokensPerSec": 324,
            "ttftMs": 639
          },
          {
            "line": 230,
            "rx": "2026-10-31T23:05:25.127Z",
            "sessionId": "2edbcac5-b483-4730-846a-a5a2ea25e0b6",
            "source": "first-token-detail",
            "tokensPerSec": 765,
            "ttftMs": 7137
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-petshop",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "petshop",
        "repetition": 3
      },
      "trialId": "petshop-mlx-qwen3-8-27b-q4-2026-10-31T23-20-23-986Z-iqxn",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-10-31T23-20-23-986Z-iqxn/recording/chat-events.jsonl",
      "recordingSha256": "9b810a8266fce883d2a8762cc9303bcbc6fcc2a4cebaa8e49541b1f260d451d0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-petshop/petshop/petshop-mlx-qwen3-8-27b-q4-2026-10-31T23-20-23-986Z-iqxn/metrics.json",
      "metricsSha256": "9d7ddec79f96ceec5c7e4eb164b352637b397cbb00d68bed544dbf2c4d4dbaf0",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 483.5,
        "sampleCount": 8,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 8,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-10-31T23:20:55.563Z",
            "sessionId": "b7feca1d-30d4-498a-baad-1e7eba0500a2",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 29905
          },
          {
            "line": 151,
            "rx": "2026-10-31T23:21:16.647Z",
            "sessionId": "b7feca1d-30d4-498a-baad-1e7eba0500a2",
            "source": "first-token-detail",
            "tokensPerSec": 300,
            "ttftMs": 1450
          },
          {
            "line": 367,
            "rx": "2026-10-31T23:22:26.399Z",
            "sessionId": "49afc51a-741b-4466-852b-4b516ab2c7db",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 35773
          },
          {
            "line": 541,
            "rx": "2026-10-31T23:23:00.197Z",
            "sessionId": "80296822-12ef-4698-9145-a0dbbf3e083e",
            "source": "first-token-detail",
            "tokensPerSec": 487,
            "ttftMs": 24075
          },
          {
            "line": 2145,
            "rx": "2026-10-31T23:27:36.997Z",
            "sessionId": "33a41b55-70e9-4a2a-9bec-705ce2b12cf9",
            "source": "first-token-detail",
            "tokensPerSec": 487,
            "ttftMs": 24861
          },
          {
            "line": 5710,
            "rx": "2026-10-31T23:37:31.916Z",
            "sessionId": "33a41b55-70e9-4a2a-9bec-705ce2b12cf9",
            "source": "first-token-detail",
            "tokensPerSec": 333,
            "ttftMs": 3226
          },
          {
            "line": 6904,
            "rx": "2026-10-31T23:40:57.672Z",
            "sessionId": "33a41b55-70e9-4a2a-9bec-705ce2b12cf9",
            "source": "first-token-detail",
            "tokensPerSec": 379,
            "ttftMs": 3219
          },
          {
            "line": 8139,
            "rx": "2026-10-31T23:44:24.054Z",
            "sessionId": "4af534db-167f-4c5f-a404-e24b07d623fe",
            "source": "first-token-detail",
            "tokensPerSec": 480,
            "ttftMs": 24423
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-petshop",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "petshop",
        "repetition": 3
      },
      "trialId": "petshop-mlx-gemma4-12b-q4-2026-10-31T23-54-38-681Z-5v0l",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-10-31T23-54-38-681Z-5v0l/recording/chat-events.jsonl",
      "recordingSha256": "b029724c0d47b1a213de54e4e69bafae909d8e50acbf507c985e2a90950389f3",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-petshop/petshop/petshop-mlx-gemma4-12b-q4-2026-10-31T23-54-38-681Z-5v0l/metrics.json",
      "metricsSha256": "aeafbdb645b3d39f1b1964437bbfab31de27b2f78298995a2304dc193054b7f1",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 603,
        "sampleCount": 34,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 34,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 24,
            "rx": "2026-10-31T23:54:55.867Z",
            "sessionId": "303f78b3-7103-452b-8cea-9a58d639621b",
            "source": "first-token-detail",
            "tokensPerSec": 1097,
            "ttftMs": 15501
          },
          {
            "line": 147,
            "rx": "2026-10-31T23:55:32.302Z",
            "sessionId": "05a5ff65-8414-403f-8f6e-13fbdd9d1f0e",
            "source": "first-token-detail",
            "tokensPerSec": 1260,
            "ttftMs": 12066
          },
          {
            "line": 282,
            "rx": "2026-10-31T23:55:56.386Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 1114,
            "ttftMs": 4478
          },
          {
            "line": 319,
            "rx": "2026-10-31T23:56:15.532Z",
            "sessionId": "05a5ff65-8414-403f-8f6e-13fbdd9d1f0e",
            "source": "first-token-detail",
            "tokensPerSec": 1118,
            "ttftMs": 11885
          },
          {
            "line": 1708,
            "rx": "2026-11-01T00:00:17.898Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 995,
            "ttftMs": 10776
          },
          {
            "line": 2222,
            "rx": "2026-11-01T00:01:32.894Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 655,
            "ttftMs": 1063
          },
          {
            "line": 2274,
            "rx": "2026-11-01T00:01:52.251Z",
            "sessionId": "5a72b036-f8b1-42ee-93ef-1759e444c19c",
            "source": "first-token-detail",
            "tokensPerSec": 962,
            "ttftMs": 11771
          },
          {
            "line": 2627,
            "rx": "2026-11-01T00:02:44.776Z",
            "sessionId": "05a5ff65-8414-403f-8f6e-13fbdd9d1f0e",
            "source": "first-token-detail",
            "tokensPerSec": 291,
            "ttftMs": 534
          },
          {
            "line": 2799,
            "rx": "2026-11-01T00:03:10.699Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 459,
            "ttftMs": 704
          },
          {
            "line": 3073,
            "rx": "2026-11-01T00:03:50.201Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 1127
          },
          {
            "line": 3115,
            "rx": "2026-11-01T00:04:03.456Z",
            "sessionId": "04339e8e-e67a-4f78-b031-2c252cd24180",
            "source": "first-token-detail",
            "tokensPerSec": 921,
            "ttftMs": 5450
          },
          {
            "line": 3171,
            "rx": "2026-11-01T00:04:20.078Z",
            "sessionId": "8743d729-49d2-4994-9e58-7a714fd4af4f",
            "source": "first-token-detail",
            "tokensPerSec": 949,
            "ttftMs": 4044
          },
          {
            "line": 3316,
            "rx": "2026-11-01T00:04:59.381Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 441,
            "ttftMs": 436
          },
          {
            "line": 3613,
            "rx": "2026-11-01T00:05:38.882Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 628,
            "ttftMs": 1639
          },
          {
            "line": 3854,
            "rx": "2026-11-01T00:06:09.775Z",
            "sessionId": "5a72b036-f8b1-42ee-93ef-1759e444c19c",
            "source": "first-token-detail",
            "tokensPerSec": 9.2,
            "ttftMs": 32774
          },
          {
            "line": 5555,
            "rx": "2026-11-01T00:10:37.308Z",
            "sessionId": "05a5ff65-8414-403f-8f6e-13fbdd9d1f0e",
            "source": "first-token-detail",
            "tokensPerSec": 497,
            "ttftMs": 677
          },
          {
            "line": 5620,
            "rx": "2026-11-01T00:10:46.357Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 678,
            "ttftMs": 1022
          },
          {
            "line": 5801,
            "rx": "2026-11-01T00:11:35.683Z",
            "sessionId": "8743d729-49d2-4994-9e58-7a714fd4af4f",
            "source": "first-token-detail",
            "tokensPerSec": 968,
            "ttftMs": 6706
          },
          {
            "line": 5973,
            "rx": "2026-11-01T00:12:22.692Z",
            "sessionId": "5a72b036-f8b1-42ee-93ef-1759e444c19c",
            "source": "first-token-detail",
            "tokensPerSec": 422,
            "ttftMs": 379
          },
          {
            "line": 6118,
            "rx": "2026-11-01T00:12:41.174Z",
            "sessionId": "05a5ff65-8414-403f-8f6e-13fbdd9d1f0e",
            "source": "first-token-detail",
            "tokensPerSec": 307,
            "ttftMs": 480
          },
          {
            "line": 6269,
            "rx": "2026-11-01T00:13:02.260Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 617,
            "ttftMs": 1096
          },
          {
            "line": 6461,
            "rx": "2026-11-01T00:13:52.105Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 528,
            "ttftMs": 790
          },
          {
            "line": 6691,
            "rx": "2026-11-01T00:14:26.345Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 864,
            "ttftMs": 2857
          },
          {
            "line": 7158,
            "rx": "2026-11-01T00:15:33.811Z",
            "sessionId": "8743d729-49d2-4994-9e58-7a714fd4af4f",
            "source": "first-token-detail",
            "tokensPerSec": 101,
            "ttftMs": 71197
          },
          {
            "line": 7346,
            "rx": "2026-11-01T00:16:27.926Z",
            "sessionId": "5a72b036-f8b1-42ee-93ef-1759e444c19c",
            "source": "first-token-detail",
            "tokensPerSec": 455,
            "ttftMs": 685
          },
          {
            "line": 7391,
            "rx": "2026-11-01T00:16:33.793Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 624,
            "ttftMs": 1798
          },
          {
            "line": 7815,
            "rx": "2026-11-01T00:17:36.926Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 581,
            "ttftMs": 1340
          },
          {
            "line": 8005,
            "rx": "2026-11-01T00:18:37.778Z",
            "sessionId": "8743d729-49d2-4994-9e58-7a714fd4af4f",
            "source": "first-token-detail",
            "tokensPerSec": 970,
            "ttftMs": 9805
          },
          {
            "line": 8189,
            "rx": "2026-11-01T00:19:32.346Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 589,
            "ttftMs": 1155
          },
          {
            "line": 8308,
            "rx": "2026-11-01T00:19:59.776Z",
            "sessionId": "1ab9e184-2435-409a-b8ed-4b05379c424a",
            "source": "first-token-detail",
            "tokensPerSec": 480,
            "ttftMs": 637
          },
          {
            "line": 8343,
            "rx": "2026-11-01T00:20:04.005Z",
            "sessionId": "4fe2b087-dd86-4ec1-b6ba-1aa78796e08d",
            "source": "first-token-detail",
            "tokensPerSec": 525,
            "ttftMs": 1302
          },
          {
            "line": 8657,
            "rx": "2026-11-01T00:21:02.036Z",
            "sessionId": "5a72b036-f8b1-42ee-93ef-1759e444c19c",
            "source": "first-token-detail",
            "tokensPerSec": 472,
            "ttftMs": 683
          },
          {
            "line": 8722,
            "rx": "2026-11-01T00:21:20.229Z",
            "sessionId": "8743d729-49d2-4994-9e58-7a714fd4af4f",
            "source": "first-token-detail",
            "tokensPerSec": 965,
            "ttftMs": 11013
          },
          {
            "line": 8849,
            "rx": "2026-11-01T00:21:58.209Z",
            "sessionId": "5a72b036-f8b1-42ee-93ef-1759e444c19c",
            "source": "first-token-detail",
            "tokensPerSec": 337,
            "ttftMs": 470
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-tankcombat",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "tankcombat",
        "repetition": 3
      },
      "trialId": "tankcombat-mlx-qwen3-8-27b-q4-2026-11-01T00-22-26-542Z-pzr8",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-11-01T00-22-26-542Z-pzr8/recording/chat-events.jsonl",
      "recordingSha256": "5442757280fa00a35df00601de5ff3e9a458f5c5f78c6dc1607031a50c878553",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-tankcombat/tankcombat/tankcombat-mlx-qwen3-8-27b-q4-2026-11-01T00-22-26-542Z-pzr8/metrics.json",
      "metricsSha256": "21f552aeb27bc59a6f4fc5834dbc938835a38340e49e853c83830a8a6ee598a7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 544.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-11-01T00:22:57.086Z",
            "sessionId": "c240cc69-17e1-4efc-a7f2-4357eaea7dea",
            "source": "first-token-detail",
            "tokensPerSec": 540,
            "ttftMs": 28853
          },
          {
            "line": 354,
            "rx": "2026-11-01T00:23:55.013Z",
            "sessionId": "e54f4dd5-7440-47ce-b883-0f6c93557437",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 8399
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-tankcombat",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "tankcombat",
        "repetition": 3
      },
      "trialId": "tankcombat-mlx-gemma4-12b-q4-2026-11-01T00-30-52-296Z-lgzo",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-11-01T00-30-52-296Z-lgzo/recording/chat-events.jsonl",
      "recordingSha256": "b3421d3c31ef7509609732d502316dc402b9640676c0f924faca42f3eac661f9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-tankcombat/tankcombat/tankcombat-mlx-gemma4-12b-q4-2026-11-01T00-30-52-296Z-lgzo/metrics.json",
      "metricsSha256": "fa110453befc3ef1492810fb82a6c420d22fa9531a813f8aceb669f4e28ae80f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1463.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 24,
            "rx": "2026-11-01T00:31:08.687Z",
            "sessionId": "bd5aabd8-03e4-4e39-bb2b-cb902282a8c6",
            "source": "first-token-detail",
            "tokensPerSec": 1235,
            "ttftMs": 14604
          },
          {
            "line": 211,
            "rx": "2026-11-01T00:31:47.708Z",
            "sessionId": "779795a8-cdfc-4b1e-adbd-7d7a997dee81",
            "source": "first-token-detail",
            "tokensPerSec": 1692,
            "ttftMs": 3672
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-schema-migration",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "schema-migration",
        "repetition": 3
      },
      "trialId": "schema-migration-mlx-gemma4-12b-q4-2026-11-01T00-38-59-115Z-0pur",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-11-01T00-38-59-115Z-0pur/recording/chat-events.jsonl",
      "recordingSha256": "6bcd0bca752a40e27d6e958c67a2d6496b7dc4aad5b37aad66ebd39b9d87212d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-schema-migration/schema-migration/schema-migration-mlx-gemma4-12b-q4-2026-11-01T00-38-59-115Z-0pur/metrics.json",
      "metricsSha256": "ecbdc1bfd6506d17019740ac7c4fb06af6204611c6a96e02a7e4957f67a4b864",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1011.5,
        "sampleCount": 12,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 12,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T00:39:12.154Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 990,
            "ttftMs": 11393
          },
          {
            "line": 144,
            "rx": "2026-11-01T00:39:41.985Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 921,
            "ttftMs": 5861
          },
          {
            "line": 167,
            "rx": "2026-11-01T00:39:52.203Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 1055,
            "ttftMs": 6236
          },
          {
            "line": 196,
            "rx": "2026-11-01T00:40:01.456Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 1360,
            "ttftMs": 883
          },
          {
            "line": 225,
            "rx": "2026-11-01T00:40:22.558Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 1131,
            "ttftMs": 6910
          },
          {
            "line": 239,
            "rx": "2026-11-01T00:40:26.359Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 1195,
            "ttftMs": 649
          },
          {
            "line": 257,
            "rx": "2026-11-01T00:40:31.579Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 1180,
            "ttftMs": 819
          },
          {
            "line": 286,
            "rx": "2026-11-01T00:40:55.724Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 1033,
            "ttftMs": 9891
          },
          {
            "line": 347,
            "rx": "2026-11-01T00:41:43.261Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 884,
            "ttftMs": 14510
          },
          {
            "line": 459,
            "rx": "2026-11-01T00:42:06.599Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 456,
            "ttftMs": 426
          },
          {
            "line": 868,
            "rx": "2026-11-01T00:43:24.171Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 696
          },
          {
            "line": 1119,
            "rx": "2026-11-01T00:44:15.637Z",
            "sessionId": "538d044c-1bbc-4f00-9b99-5ef24a0fd305",
            "source": "first-token-detail",
            "tokensPerSec": 982,
            "ttftMs": 13563
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-schema-migration",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "schema-migration",
        "repetition": 3
      },
      "trialId": "schema-migration-mlx-qwen3-8-27b-q4-2026-11-01T00-48-22-682Z-llwn",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-11-01T00-48-22-682Z-llwn/recording/chat-events.jsonl",
      "recordingSha256": "614be9c936f07aaaca1bfec23fba57c570d503cb1acf69203217c16d75d126cf",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-schema-migration/schema-migration/schema-migration-mlx-qwen3-8-27b-q4-2026-11-01T00-48-22-682Z-llwn/metrics.json",
      "metricsSha256": "a0bfea74e4827392979b1770e6ff7799349af5a3b6c53267792c0975954fc7af",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 540,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-11-01T00:48:38.875Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 533,
            "ttftMs": 14542
          },
          {
            "line": 93,
            "rx": "2026-11-01T00:49:01.146Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 540,
            "ttftMs": 7548
          },
          {
            "line": 134,
            "rx": "2026-11-01T00:49:17.799Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 644,
            "ttftMs": 8200
          },
          {
            "line": 179,
            "rx": "2026-11-01T00:49:25.429Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 579,
            "ttftMs": 1272
          },
          {
            "line": 235,
            "rx": "2026-11-01T00:49:35.149Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 463,
            "ttftMs": 928
          },
          {
            "line": 455,
            "rx": "2026-11-01T00:50:23.169Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 538,
            "ttftMs": 8599
          },
          {
            "line": 576,
            "rx": "2026-11-01T00:51:04.476Z",
            "sessionId": "04b32b8c-6bef-401e-acca-07e4d3e67ac4",
            "source": "first-token-detail",
            "tokensPerSec": 545,
            "ttftMs": 14593
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-failing-tests-spec",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "failing-tests-spec",
        "repetition": 3
      },
      "trialId": "failing-tests-spec-mlx-gemma4-12b-q4-2026-11-01T00-52-01-218Z-qqzj",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-11-01T00-52-01-218Z-qqzj/recording/chat-events.jsonl",
      "recordingSha256": "dc8accbb0e5e2ad35b45ca67e276c52662e5d3ec60dd0b82956132d063c51123",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-gemma4-12b-q4-2026-11-01T00-52-01-218Z-qqzj/metrics.json",
      "metricsSha256": "329efb40262f035ee8885dec6718760f1d02767290e1f93e84e5265f0ed97d8f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1131.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-01T00:52:13.431Z",
            "sessionId": "a08f84ff-698a-4e04-b744-1e7feb6606c6",
            "source": "first-token-detail",
            "tokensPerSec": 1096,
            "ttftMs": 10435
          },
          {
            "line": 195,
            "rx": "2026-11-01T00:53:20.237Z",
            "sessionId": "a08f84ff-698a-4e04-b744-1e7feb6606c6",
            "source": "first-token-detail",
            "tokensPerSec": 1254,
            "ttftMs": 6506
          },
          {
            "line": 510,
            "rx": "2026-11-01T00:54:54.170Z",
            "sessionId": "a08f84ff-698a-4e04-b744-1e7feb6606c6",
            "source": "first-token-detail",
            "tokensPerSec": 1138,
            "ttftMs": 10050
          },
          {
            "line": 707,
            "rx": "2026-11-01T00:56:10.998Z",
            "sessionId": "a08f84ff-698a-4e04-b744-1e7feb6606c6",
            "source": "first-token-detail",
            "tokensPerSec": 1125,
            "ttftMs": 1000
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-failing-tests-spec",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "failing-tests-spec",
        "repetition": 3
      },
      "trialId": "failing-tests-spec-mlx-qwen3-8-27b-q4-2026-11-01T00-57-49-886Z-4wf3",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-11-01T00-57-49-886Z-4wf3/recording/chat-events.jsonl",
      "recordingSha256": "e538bf1be15e1586d59d4761fbe8249054e635efb32c32698a08753a4e244df4",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-mlx-qwen3-8-27b-q4-2026-11-01T00-57-49-886Z-4wf3/metrics.json",
      "metricsSha256": "28d36638d8589ca10bee899d6a5042addad92ea14ec7a42eecf1c25f006c1de3",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 578,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-01T00:58:04.335Z",
            "sessionId": "7c97d83b-88ed-4a8f-8ce3-d351928b55f7",
            "source": "first-token-detail",
            "tokensPerSec": 578,
            "ttftMs": 12767
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-symptom-debug",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "symptom-debug",
        "repetition": 3
      },
      "trialId": "symptom-debug-mlx-qwen3-8-27b-q4-2026-11-01T01-01-01-013Z-33md",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-11-01T01-01-01-013Z-33md/recording/chat-events.jsonl",
      "recordingSha256": "b6b1cab229df1d6fb17a320e1c2c07d67860b90e1329bc7e940a8b9da5df922f",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-qwen3-8-27b-q4-2026-11-01T01-01-01-013Z-33md/metrics.json",
      "metricsSha256": "043bd2bd5a237a4befc1032ecd18e45b14fac7b43218ece177abe638109d25fe",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 673,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T01:01:15.343Z",
            "sessionId": "48a75e3a-978d-4976-8b01-38d061ca1ded",
            "source": "first-token-detail",
            "tokensPerSec": 673,
            "ttftMs": 12668
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-symptom-debug",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "symptom-debug",
        "repetition": 3
      },
      "trialId": "symptom-debug-mlx-gemma4-12b-q4-2026-11-01T01-03-16-564Z-5fxr",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-11-01T01-03-16-564Z-5fxr/recording/chat-events.jsonl",
      "recordingSha256": "3cd6b0c7818fc2d50a3c0ef317524e8d828871ff92cc6218fe29a2586dab0dc1",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-symptom-debug/symptom-debug/symptom-debug-mlx-gemma4-12b-q4-2026-11-01T01-03-16-564Z-5fxr/metrics.json",
      "metricsSha256": "01531f5f39e1ea75594f78a562eaa3e80c0b4cd602cb964d8c2b6d3815193045",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1162,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-01T01:03:28.187Z",
            "sessionId": "0ec6f809-328f-4b44-a038-e59120214c1e",
            "source": "first-token-detail",
            "tokensPerSec": 1358,
            "ttftMs": 9986
          },
          {
            "line": 1438,
            "rx": "2026-11-01T01:07:22.741Z",
            "sessionId": "0ec6f809-328f-4b44-a038-e59120214c1e",
            "source": "first-token-detail",
            "tokensPerSec": 966,
            "ttftMs": 15219
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-data-wrangle",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "data-wrangle",
        "repetition": 3
      },
      "trialId": "data-wrangle-mlx-qwen3-8-27b-q4-2026-11-01T01-07-51-241Z-aqft",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-11-01T01-07-51-241Z-aqft/recording/chat-events.jsonl",
      "recordingSha256": "e0fd8f8c849bf0d1199acc76a2d344de6dff74463a320f5f44017a3a6a659f18",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-qwen3-8-27b-q4-2026-11-01T01-07-51-241Z-aqft/metrics.json",
      "metricsSha256": "ec39d5b84920f5291c16c300929981e7bae7e4c4f68aab5af446a97def06da2b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 516,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T01:08:06.947Z",
            "sessionId": "163e4508-5597-4522-b366-06e6f647e68f",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 14067
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-data-wrangle",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "data-wrangle",
        "repetition": 3
      },
      "trialId": "data-wrangle-mlx-gemma4-12b-q4-2026-11-01T01-14-44-501Z-5b5h",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-11-01T01-14-44-501Z-5b5h/recording/chat-events.jsonl",
      "recordingSha256": "b4bc5530bab5c8de5a82366e8a136a8c88823caeb9e72f3f7f1cf91b9c6432b6",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-data-wrangle/data-wrangle/data-wrangle-mlx-gemma4-12b-q4-2026-11-01T01-14-44-501Z-5b5h/metrics.json",
      "metricsSha256": "1c5cd29cd4f5e018cbacb1f90e5786732bfbec7fe9c028a8b976de6f6f05a784",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1080,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-01T01:14:57.254Z",
            "sessionId": "98a5860d-b8ee-4d1d-b10b-ad196b8bce4a",
            "source": "first-token-detail",
            "tokensPerSec": 1080,
            "ttftMs": 11120
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-incident-postmortem",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "incident-postmortem",
        "repetition": 3
      },
      "trialId": "incident-postmortem-mlx-gemma4-12b-q4-2026-11-01T01-25-52-129Z-1hb1",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-11-01T01-25-52-129Z-1hb1/recording/chat-events.jsonl",
      "recordingSha256": "6cfa570155c8a84a31aa5669890dfa2f89228fc6cc75381938cbb40cc0c3db36",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-gemma4-12b-q4-2026-11-01T01-25-52-129Z-1hb1/metrics.json",
      "metricsSha256": "0471eeed3935d3e8d1eb16a03510e230dcce16c3c94e35ae3a3e59b7c81a7a7c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1016,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-11-01T01:26:09.981Z",
            "sessionId": "819a0d8a-d515-42b4-8379-d8b5587b5d1b",
            "source": "first-token-detail",
            "tokensPerSec": 1032,
            "ttftMs": 16215
          },
          {
            "line": 1144,
            "rx": "2026-11-01T01:30:01.066Z",
            "sessionId": "819a0d8a-d515-42b4-8379-d8b5587b5d1b",
            "source": "first-token-detail",
            "tokensPerSec": 1165,
            "ttftMs": 11350
          },
          {
            "line": 1234,
            "rx": "2026-11-01T01:30:40.227Z",
            "sessionId": "819a0d8a-d515-42b4-8379-d8b5587b5d1b",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 14787
          },
          {
            "line": 1583,
            "rx": "2026-11-01T01:31:44.294Z",
            "sessionId": "819a0d8a-d515-42b4-8379-d8b5587b5d1b",
            "source": "first-token-detail",
            "tokensPerSec": 462,
            "ttftMs": 408
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-incident-postmortem",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "incident-postmortem",
        "repetition": 3
      },
      "trialId": "incident-postmortem-mlx-qwen3-8-27b-q4-2026-11-01T01-35-14-261Z-7yx3",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-11-01T01-35-14-261Z-7yx3/recording/chat-events.jsonl",
      "recordingSha256": "c83ab6abba8e773c18538423ceda4e103978fc111b44bd2fa7487d912e0438af",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-incident-postmortem/incident-postmortem/incident-postmortem-mlx-qwen3-8-27b-q4-2026-11-01T01-35-14-261Z-7yx3/metrics.json",
      "metricsSha256": "7321e6a2f6af44eedcde138beec6d8a7125ab974d478668332fe065dbe640d94",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 553,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T01:35:42.881Z",
            "sessionId": "1173b4ad-7ca3-461d-adf2-c4c7998981b3",
            "source": "first-token-detail",
            "tokensPerSec": 553,
            "ttftMs": 26981
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-ops-runbook-anomaly",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "ops-runbook-anomaly",
        "repetition": 3
      },
      "trialId": "ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-11-01T01-42-55-909Z-v8kh",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-11-01T01-42-55-909Z-v8kh/recording/chat-events.jsonl",
      "recordingSha256": "6c92ef1ccdd209bb2f8c303bb24038835cc28fbf007d4884bd5ae20e96249158",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-gemma4-12b-q4-2026-11-01T01-42-55-909Z-v8kh/metrics.json",
      "metricsSha256": "16926c016d3898140928b0307fba169f04bdce97b456e6b8ee20111daad4a026",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1179,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T01:43:13.467Z",
            "sessionId": "1d2c5c63-a04f-4258-a907-a64280c06eb0",
            "source": "first-token-detail",
            "tokensPerSec": 979,
            "ttftMs": 16026
          },
          {
            "line": 187,
            "rx": "2026-11-01T01:43:47.776Z",
            "sessionId": "1d2c5c63-a04f-4258-a907-a64280c06eb0",
            "source": "first-token-detail",
            "tokensPerSec": 1238,
            "ttftMs": 5106
          },
          {
            "line": 232,
            "rx": "2026-11-01T01:44:07.722Z",
            "sessionId": "1d2c5c63-a04f-4258-a907-a64280c06eb0",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 5442
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-ops-runbook-anomaly",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "ops-runbook-anomaly",
        "repetition": 3
      },
      "trialId": "ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-11-01T01-46-56-510Z-4f6u",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-11-01T01-46-56-510Z-4f6u/recording/chat-events.jsonl",
      "recordingSha256": "50178663b55cc2e0269bfc9d147369d04795e00165fbef630f4e14d716c5a2bd",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-mlx-qwen3-8-27b-q4-2026-11-01T01-46-56-510Z-4f6u/metrics.json",
      "metricsSha256": "78eeea0ee53b34f1363825c3127d587eb97c38cd2f29368aa40302578a67c1fc",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 587,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T01:47:24.021Z",
            "sessionId": "67295178-12b9-4397-beaf-a463e4c38f98",
            "source": "first-token-detail",
            "tokensPerSec": 587,
            "ttftMs": 25862
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-plan-and-estimate",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "plan-and-estimate",
        "repetition": 3
      },
      "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T01-53-48-630Z-lhjf",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T01-53-48-630Z-lhjf/recording/chat-events.jsonl",
      "recordingSha256": "6e4a26ed4d709dca34d19fed2ca8eb51af8e94eec8aa0b432245fd1d146bc87e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T01-53-48-630Z-lhjf/metrics.json",
      "metricsSha256": "976a75fa4eb4c2a812295c377e808a508435aa0b8c5259e50f8c6b3f551acac9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 552,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T01:54:04.905Z",
            "sessionId": "242db5bb-b600-417c-9521-e37813c806fa",
            "source": "first-token-detail",
            "tokensPerSec": 552,
            "ttftMs": 14631
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-plan-and-estimate",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "plan-and-estimate",
        "repetition": 3
      },
      "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T01-57-17-006Z-4xa8",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T01-57-17-006Z-4xa8/recording/chat-events.jsonl",
      "recordingSha256": "5ac63e510277ce5e5ec9b5f89536efefd5a25ca4b5a107370603d04442b7bf91",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T01-57-17-006Z-4xa8/metrics.json",
      "metricsSha256": "74ae273017d5dc713328ac8451c3b964ae79cb081275664b2f6c2d11bce2a411",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1319,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-11-01T01:57:28.915Z",
            "sessionId": "b9c316a3-416e-438c-9866-b30159595562",
            "source": "first-token-detail",
            "tokensPerSec": 1319,
            "ttftMs": 10274
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-core-conflict-synthesis",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "core",
        "scenario": "conflict-synthesis",
        "repetition": 3
      },
      "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T01-58-17-534Z-e91g",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T01-58-17-534Z-e91g/recording/chat-events.jsonl",
      "recordingSha256": "f47457fa2f264f24a3ffb473d08b03d46747a219e71379bf8c4f0d230cb99c9b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T01-58-17-534Z-e91g/metrics.json",
      "metricsSha256": "0ab2fce0a38084c9820c07c007e5c2ab953c8e649c9ead63f126c3ab64844430",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 614.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T01:58:42.764Z",
            "sessionId": "7d44638a-8ca6-488f-bd72-880c3313afb1",
            "source": "first-token-detail",
            "tokensPerSec": 647,
            "ttftMs": 23540
          },
          {
            "line": 661,
            "rx": "2026-11-01T02:00:33.171Z",
            "sessionId": "7d44638a-8ca6-488f-bd72-880c3313afb1",
            "source": "first-token-detail",
            "tokensPerSec": 582,
            "ttftMs": 8496
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-core-conflict-synthesis",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "core",
        "scenario": "conflict-synthesis",
        "repetition": 3
      },
      "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T02-14-16-657Z-7y28",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T02-14-16-657Z-7y28/recording/chat-events.jsonl",
      "recordingSha256": "0753a62f7d5175e732295313ac045be0b47b53b54b9cc05f2362d69f36b46dc7",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T02-14-16-657Z-7y28/metrics.json",
      "metricsSha256": "6492a4e442172e7886903801b5b9f57dc0045a886a555f7ce6093fa9a1cc340e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1180.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-11-01T02:14:33.856Z",
            "sessionId": "90bf4eea-bfca-4e4e-b3d2-33f027eeb1e1",
            "source": "first-token-detail",
            "tokensPerSec": 1078,
            "ttftMs": 15553
          },
          {
            "line": 166,
            "rx": "2026-11-01T02:15:07.669Z",
            "sessionId": "90bf4eea-bfca-4e4e-b3d2-33f027eeb1e1",
            "source": "first-token-detail",
            "tokensPerSec": 1427,
            "ttftMs": 4117
          },
          {
            "line": 253,
            "rx": "2026-11-01T02:15:29.112Z",
            "sessionId": "90bf4eea-bfca-4e4e-b3d2-33f027eeb1e1",
            "source": "first-token-detail",
            "tokensPerSec": 1278,
            "ttftMs": 6011
          },
          {
            "line": 298,
            "rx": "2026-11-01T02:15:49.490Z",
            "sessionId": "90bf4eea-bfca-4e4e-b3d2-33f027eeb1e1",
            "source": "first-token-detail",
            "tokensPerSec": 1083,
            "ttftMs": 6309
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-constrained-comms",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "constrained-comms",
        "repetition": 3
      },
      "trialId": "constrained-comms-mlx-gemma4-12b-q4-2026-11-01T02-19-15-016Z-38lk",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-11-01T02-19-15-016Z-38lk/recording/chat-events.jsonl",
      "recordingSha256": "1f062ef321a6da8e4fc0ac98b164b0a9335eb24731d0892fcf41edda60870c98",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-gemma4-12b-q4-2026-11-01T02-19-15-016Z-38lk/metrics.json",
      "metricsSha256": "02cf5570eaa523fbdfd5c8375046ab96e800c1e2a5db4191f479ffd83fe8a436",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1584,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T02:19:30.556Z",
            "sessionId": "7baf0a4d-e8ba-4f37-ac92-821e2a8005e0",
            "source": "first-token-detail",
            "tokensPerSec": 1303,
            "ttftMs": 13913
          },
          {
            "line": 103,
            "rx": "2026-11-01T02:19:55.011Z",
            "sessionId": "7baf0a4d-e8ba-4f37-ac92-821e2a8005e0",
            "source": "first-token-detail",
            "tokensPerSec": 1865,
            "ttftMs": 3149
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-constrained-comms",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "constrained-comms",
        "repetition": 3
      },
      "trialId": "constrained-comms-mlx-qwen3-8-27b-q4-2026-11-01T02-21-20-714Z-y3cq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-11-01T02-21-20-714Z-y3cq/recording/chat-events.jsonl",
      "recordingSha256": "9ebb3285038f352e88e0f63a77d406ccfbb030b110a2c3cfb86f3b86e65dab2f",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-constrained-comms/constrained-comms/constrained-comms-mlx-qwen3-8-27b-q4-2026-11-01T02-21-20-714Z-y3cq/metrics.json",
      "metricsSha256": "57a7ebf60bb1052c3ea1a63f0319a935066de4ccd9569c1ab3a8deb10efe886f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 798,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T02:21:45.953Z",
            "sessionId": "85506824-1a9e-4da9-bc34-c7fb24dad170",
            "source": "first-token-detail",
            "tokensPerSec": 653,
            "ttftMs": 23599
          },
          {
            "line": 85,
            "rx": "2026-11-01T02:22:08.159Z",
            "sessionId": "85506824-1a9e-4da9-bc34-c7fb24dad170",
            "source": "first-token-detail",
            "tokensPerSec": 820,
            "ttftMs": 5602
          },
          {
            "line": 132,
            "rx": "2026-11-01T02:22:25.855Z",
            "sessionId": "85506824-1a9e-4da9-bc34-c7fb24dad170",
            "source": "first-token-detail",
            "tokensPerSec": 798,
            "ttftMs": 3732
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-records-intake",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "records-intake",
        "repetition": 3
      },
      "trialId": "records-intake-mlx-gemma4-12b-q4-2026-11-01T02-23-00-672Z-jeky",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-11-01T02-23-00-672Z-jeky/recording/chat-events.jsonl",
      "recordingSha256": "b583ca345689ca184dbce7849b365a4f2b1c95c252158828a31625b87238a70a",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-records-intake/records-intake/records-intake-mlx-gemma4-12b-q4-2026-11-01T02-23-00-672Z-jeky/metrics.json",
      "metricsSha256": "fbc8fae6979fd3f2d8c57edc8c5e6d79d328b3cf7aa41fa3738d36f4bacd3bce",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1333,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-11-01T02:23:12.418Z",
            "sessionId": "a4e40abc-57fd-4a97-8d56-fd0cccd85064",
            "source": "first-token-detail",
            "tokensPerSec": 1333,
            "ttftMs": 10120
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-records-intake",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "records-intake",
        "repetition": 3
      },
      "trialId": "records-intake-mlx-qwen3-8-27b-q4-2026-11-01T02-26-47-094Z-myp0",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-11-01T02-26-47-094Z-myp0/recording/chat-events.jsonl",
      "recordingSha256": "8ebdde73ac7be143bbbaf834c127e42491763202f13071b455c1eff21ed1bbaa",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-records-intake/records-intake/records-intake-mlx-qwen3-8-27b-q4-2026-11-01T02-26-47-094Z-myp0/metrics.json",
      "metricsSha256": "ca2206e9c27e86a0fc7b30e7ad6bbe2a1096629d85c944e4b52afb6d9ebfabf9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 699,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T02:27:01.652Z",
            "sessionId": "c16726df-792c-4118-993f-6de774e5f434",
            "source": "first-token-detail",
            "tokensPerSec": 699,
            "ttftMs": 13029
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-plan-and-estimate",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "plan-and-estimate",
        "repetition": 3
      },
      "trialId": "plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T02-36-36-540Z-1pas",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T02-36-36-540Z-1pas/recording/chat-events.jsonl",
      "recordingSha256": "2d132d189b3b5c984d5a4e315ebd0acf2c8ba6f270224cd59d104e0b12c34c6d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-qwen3-8-27b-q4-2026-11-01T02-36-36-540Z-1pas/metrics.json",
      "metricsSha256": "1bfc73993f3470b3526818041943fc4a44b9b0c087be289c979bb3bfaca9af57",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 557,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T02:36:52.965Z",
            "sessionId": "64bb1801-fc73-434a-b750-53e58f0bdc40",
            "source": "first-token-detail",
            "tokensPerSec": 544,
            "ttftMs": 14769
          },
          {
            "line": 598,
            "rx": "2026-11-01T02:38:37.743Z",
            "sessionId": "64bb1801-fc73-434a-b750-53e58f0bdc40",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 9114
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-plan-and-estimate",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "plan-and-estimate",
        "repetition": 3
      },
      "trialId": "plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T02-42-06-414Z-46bn",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T02-42-06-414Z-46bn/recording/chat-events.jsonl",
      "recordingSha256": "3d60ba58dccb2b0c9d7cf84bb5ba9903a9bb0bf48fb7150fd642bb14a3dd893d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-mlx-gemma4-12b-q4-2026-11-01T02-42-06-414Z-46bn/metrics.json",
      "metricsSha256": "6fa364962686755cbe75a78de95e6a6c35e7aee22680ea8421eb1ad2226bbe55",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1093,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-01T02:42:19.196Z",
            "sessionId": "53ba697c-d9d3-4add-ada5-e3979e1da8d0",
            "source": "first-token-detail",
            "tokensPerSec": 1093,
            "ttftMs": 11141
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-meeting-followup",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "meeting-followup",
        "repetition": 3
      },
      "trialId": "meeting-followup-mlx-qwen3-8-27b-q4-2026-11-01T02-43-15-912Z-yuuj",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-11-01T02-43-15-912Z-yuuj/recording/chat-events.jsonl",
      "recordingSha256": "b2495649d45b5a44f67a9b2dbc8c3f486bab4b92abb2d47f317ac4cb07737693",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-qwen3-8-27b-q4-2026-11-01T02-43-15-912Z-yuuj/metrics.json",
      "metricsSha256": "f686013717fe4862f462c885244018222977025138ca9fe7b8425b5f54f9f641",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 704,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-11-01T02:43:37.731Z",
            "sessionId": "b5056c32-02be-4ddf-b623-bfa15ede12e3",
            "source": "first-token-detail",
            "tokensPerSec": 564,
            "ttftMs": 20183
          },
          {
            "line": 190,
            "rx": "2026-11-01T02:45:35.546Z",
            "sessionId": "b5056c32-02be-4ddf-b623-bfa15ede12e3",
            "source": "first-token-detail",
            "tokensPerSec": 844,
            "ttftMs": 2769
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-meeting-followup",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "meeting-followup",
        "repetition": 3
      },
      "trialId": "meeting-followup-mlx-gemma4-12b-q4-2026-11-01T02-48-48-435Z-8b3g",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-11-01T02-48-48-435Z-8b3g/recording/chat-events.jsonl",
      "recordingSha256": "3d093bb6d3c15f597d8b4099bfef1378d29f3702ee5db5c2fcea25b94e5c0d55",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-meeting-followup/meeting-followup/meeting-followup-mlx-gemma4-12b-q4-2026-11-01T02-48-48-435Z-8b3g/metrics.json",
      "metricsSha256": "7d16416204d94d5da74f847a0269b4bd25ced69badcb1492108b41319546b8f4",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 927,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-11-01T02:49:00.006Z",
            "sessionId": "537bb872-e727-4cf5-9745-e5cebac576af",
            "source": "first-token-detail",
            "tokensPerSec": 1930,
            "ttftMs": 9938
          },
          {
            "line": 111,
            "rx": "2026-11-01T02:50:49.157Z",
            "sessionId": "537bb872-e727-4cf5-9745-e5cebac576af",
            "source": "first-token-detail",
            "tokensPerSec": 1935,
            "ttftMs": 3987
          },
          {
            "line": 1692,
            "rx": "2026-11-01T02:55:13.305Z",
            "sessionId": "537bb872-e727-4cf5-9745-e5cebac576af",
            "source": "first-token-detail",
            "tokensPerSec": 850,
            "ttftMs": 9753
          },
          {
            "line": 1718,
            "rx": "2026-11-01T02:55:31.260Z",
            "sessionId": "11510b80-b583-4b75-a0c2-a3fe30cce824",
            "source": "first-token-detail",
            "tokensPerSec": 812,
            "ttftMs": 5966
          },
          {
            "line": 2080,
            "rx": "2026-11-01T02:56:39.418Z",
            "sessionId": "537bb872-e727-4cf5-9745-e5cebac576af",
            "source": "first-token-detail",
            "tokensPerSec": 854,
            "ttftMs": 12758
          },
          {
            "line": 2228,
            "rx": "2026-11-01T02:57:05.775Z",
            "sessionId": "11510b80-b583-4b75-a0c2-a3fe30cce824",
            "source": "first-token-detail",
            "tokensPerSec": 927,
            "ttftMs": 5897
          },
          {
            "line": 2255,
            "rx": "2026-11-01T02:57:20.984Z",
            "sessionId": "11510b80-b583-4b75-a0c2-a3fe30cce824",
            "source": "first-token-detail",
            "tokensPerSec": 1245,
            "ttftMs": 4135
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-craftbook-spreadsheet-model",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-spreadsheet-model",
        "repetition": 3
      },
      "trialId": "craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-11-01T03-15-16-907Z-3wr5",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-11-01T03-15-16-907Z-3wr5/recording/chat-events.jsonl",
      "recordingSha256": "65af765a400d0fa6d491d2c1d61106c7071d20917cdd3bc39e7e31d4be47d845",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-gemma4-12b-q4-2026-11-01T03-15-16-907Z-3wr5/metrics.json",
      "metricsSha256": "6419b7943a6aa4fd1ea995d4723b2389e21752bbbffe38018357d8ecb00ad214",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1262,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T03:15:37.248Z",
            "sessionId": "a66885cf-d404-4174-8f95-692d098f7528",
            "source": "first-token-detail",
            "tokensPerSec": 1097,
            "ttftMs": 13831
          },
          {
            "line": 178,
            "rx": "2026-11-01T03:16:15.246Z",
            "sessionId": "078d7f5d-4c4d-46ef-9c8d-b73c2159ed74",
            "source": "first-token-detail",
            "tokensPerSec": 1380,
            "ttftMs": 6773
          },
          {
            "line": 379,
            "rx": "2026-11-01T03:17:15.419Z",
            "sessionId": "242213fd-b89f-4c97-9d88-c396b5d10e93",
            "source": "first-token-detail",
            "tokensPerSec": 1262,
            "ttftMs": 6963
          },
          {
            "line": 979,
            "rx": "2026-11-01T03:19:11.680Z",
            "sessionId": "8a579071-e9db-4f50-892d-7c5d15eb85fa",
            "source": "first-token-detail",
            "tokensPerSec": 1191,
            "ttftMs": 8194
          },
          {
            "line": 1221,
            "rx": "2026-11-01T03:20:24.342Z",
            "sessionId": "b24c9270-15d3-439c-83cb-0f1f28d091de",
            "source": "first-token-detail",
            "tokensPerSec": 1311,
            "ttftMs": 5837
          },
          {
            "line": 3150,
            "rx": "2026-11-01T03:25:55.466Z",
            "sessionId": "55999641-1b63-4ec3-959b-fedea0df7749",
            "source": "first-token-detail",
            "tokensPerSec": 1173,
            "ttftMs": 6883
          },
          {
            "line": 3240,
            "rx": "2026-11-01T03:26:14.921Z",
            "sessionId": "2d3fb5da-bfe1-460f-99fc-edb300033378",
            "source": "first-token-detail",
            "tokensPerSec": 1308,
            "ttftMs": 3888
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-craftbook-spreadsheet-model",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-spreadsheet-model",
        "repetition": 3
      },
      "trialId": "craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-11-01T03-33-17-724Z-hzpq",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-11-01T03-33-17-724Z-hzpq/recording/chat-events.jsonl",
      "recordingSha256": "5781d2d950d71ec7ea65dd201f82553272d08a6966d9763a7458ba6bdaad41fc",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-mlx-qwen3-8-27b-q4-2026-11-01T03-33-17-724Z-hzpq/metrics.json",
      "metricsSha256": "e5dc44169777bec7bda2e4c4a52f46c44c415ba6ac6d69563d1508f10bf20787",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 562,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-11-01T03:33:45.012Z",
            "sessionId": "579083f8-3ca1-47b9-8aa2-a43eaab4fbda",
            "source": "first-token-detail",
            "tokensPerSec": 562,
            "ttftMs": 20780
          },
          {
            "line": 513,
            "rx": "2026-11-01T03:35:23.257Z",
            "sessionId": "238569a3-df10-4bef-b86e-29a8d3f9d2c8",
            "source": "first-token-detail",
            "tokensPerSec": 627,
            "ttftMs": 13881
          },
          {
            "line": 2397,
            "rx": "2026-11-01T03:41:06.609Z",
            "sessionId": "2ef1983c-9eb0-4567-bade-2d8b9a189281",
            "source": "first-token-detail",
            "tokensPerSec": 511,
            "ttftMs": 17154
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-conflict-synthesis",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "conflict-synthesis",
        "repetition": 3
      },
      "trialId": "conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T03-49-29-133Z-8slx",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T03-49-29-133Z-8slx/recording/chat-events.jsonl",
      "recordingSha256": "18f97102db6cd71a2aa272e140edd4deffd96942dd63dc7ed705683764740269",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-gemma4-12b-q4-2026-11-01T03-49-29-133Z-8slx/metrics.json",
      "metricsSha256": "13afef5863f597203165d821c0af4e9b17814c4450705d264fcef0b478d46649",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1081,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-11-01T03:49:46.297Z",
            "sessionId": "a91c86fb-9b1f-4327-b72f-0783750f5692",
            "source": "first-token-detail",
            "tokensPerSec": 1081,
            "ttftMs": 15541
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-conflict-synthesis",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "conflict-synthesis",
        "repetition": 3
      },
      "trialId": "conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T03-52-56-729Z-jhrw",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T03-52-56-729Z-jhrw/recording/chat-events.jsonl",
      "recordingSha256": "a3572a1fdb3af9635a57e1671c96ff1b80134a0e50ec7396e88c6d78c9982180",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-mlx-qwen3-8-27b-q4-2026-11-01T03-52-56-729Z-jhrw/metrics.json",
      "metricsSha256": "637dbfc385d11c8efeb49ffe7e2805fe3c56f9017a7e722f5963c4fdf757f4a1",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 583,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T03:53:24.420Z",
            "sessionId": "2f81df0b-8d14-4a97-9375-5bd74d61ed26",
            "source": "first-token-detail",
            "tokensPerSec": 583,
            "ttftMs": 26042
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-docblocks-theme-roundtrip",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "docblocks-theme-roundtrip",
        "repetition": 3
      },
      "trialId": "docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-11-01T04-19-58-506Z-xpc0",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-11-01T04-19-58-506Z-xpc0/recording/chat-events.jsonl",
      "recordingSha256": "de0dbad5605b7abe17496d999032ba9e7fa078e5afd154f363da2c57105cbf74",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-qwen3-8-27b-q4-2026-11-01T04-19-58-506Z-xpc0/metrics.json",
      "metricsSha256": "f41390491439d44b9d19b3e9b7ae7c0bf043d0c033544436bb73c4f28834d174",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 537.5,
        "sampleCount": 8,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 8,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T04:20:15.185Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 526,
            "ttftMs": 15054
          },
          {
            "line": 180,
            "rx": "2026-11-01T04:20:48.550Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 454,
            "ttftMs": 7320
          },
          {
            "line": 365,
            "rx": "2026-11-01T04:21:30.207Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 509,
            "ttftMs": 10293
          },
          {
            "line": 537,
            "rx": "2026-11-01T04:22:16.714Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 609,
            "ttftMs": 6631
          },
          {
            "line": 599,
            "rx": "2026-11-01T04:22:44.836Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 9654
          },
          {
            "line": 660,
            "rx": "2026-11-01T04:23:16.547Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 11247
          },
          {
            "line": 720,
            "rx": "2026-11-01T04:23:53.885Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 557,
            "ttftMs": 13454
          },
          {
            "line": 838,
            "rx": "2026-11-01T04:24:46.550Z",
            "sessionId": "fc94ab89-dd76-416d-b8dc-ed502e9e28e1",
            "source": "first-token-detail",
            "tokensPerSec": 555,
            "ttftMs": 15945
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-docblocks-theme-roundtrip",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "docblocks-theme-roundtrip",
        "repetition": 3
      },
      "trialId": "docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-11-01T04-30-05-917Z-74to",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-11-01T04-30-05-917Z-74to/recording/chat-events.jsonl",
      "recordingSha256": "2cf6181dc035e30223bc6af72b21e5135d44720a045511b4aaf412f79bc4f5d2",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-mlx-gemma4-12b-q4-2026-11-01T04-30-05-917Z-74to/metrics.json",
      "metricsSha256": "a5336b241f53ca3c36c36d23fe031e9926eabcda5116cb193c8e03a5f52fe216",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1150.5,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-11-01T04:30:16.040Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1803,
            "ttftMs": 8394
          },
          {
            "line": 239,
            "rx": "2026-11-01T04:31:33.090Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 536
          },
          {
            "line": 1071,
            "rx": "2026-11-01T04:33:52.379Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1657,
            "ttftMs": 4349
          },
          {
            "line": 1117,
            "rx": "2026-11-01T04:34:09.330Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1134,
            "ttftMs": 6247
          },
          {
            "line": 1137,
            "rx": "2026-11-01T04:34:13.832Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 955,
            "ttftMs": 706
          },
          {
            "line": 1159,
            "rx": "2026-11-01T04:34:18.758Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1054,
            "ttftMs": 604
          },
          {
            "line": 1173,
            "rx": "2026-11-01T04:34:23.667Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1167,
            "ttftMs": 478
          },
          {
            "line": 1187,
            "rx": "2026-11-01T04:34:28.750Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1059,
            "ttftMs": 526
          },
          {
            "line": 1229,
            "rx": "2026-11-01T04:34:56.859Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1211,
            "ttftMs": 8548
          },
          {
            "line": 1261,
            "rx": "2026-11-01T04:35:23.607Z",
            "sessionId": "87b4835a-3b04-4806-a7c3-826abef25f14",
            "source": "first-token-detail",
            "tokensPerSec": 1095,
            "ttftMs": 10200
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-craftbook-powerpoint-deck",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-powerpoint-deck",
        "repetition": 3
      },
      "trialId": "craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-11-01T04-35-51-551Z-qzuu",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-11-01T04-35-51-551Z-qzuu/recording/chat-events.jsonl",
      "recordingSha256": "7f49188cca46aed6e27f32a8a9a86614799d8d1597a7b27c6a92e7e5886d3e77",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-qwen3-8-27b-q4-2026-11-01T04-35-51-551Z-qzuu/metrics.json",
      "metricsSha256": "597654554768445c73188fa0db0ed9c0c2bdf250ae7fdd804610a37615271633",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 519,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-11-01T04:36:25.178Z",
            "sessionId": "80c6676c-25f0-4dd3-a8fd-97d3c81e62f4",
            "source": "first-token-detail",
            "tokensPerSec": 505,
            "ttftMs": 27095
          },
          {
            "line": 1567,
            "rx": "2026-11-01T04:41:13.568Z",
            "sessionId": "518d414e-cb78-4834-9690-1378bdc2cba8",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 15306
          },
          {
            "line": 2147,
            "rx": "2026-11-01T04:43:14.607Z",
            "sessionId": "320422ed-fe68-4576-a977-4e5aa75e708a",
            "source": "first-token-detail",
            "tokensPerSec": 538,
            "ttftMs": 16332
          },
          {
            "line": 2357,
            "rx": "2026-11-01T04:44:13.409Z",
            "sessionId": "7ced45ac-5e72-4096-b60a-c8497d688142",
            "source": "first-token-detail",
            "tokensPerSec": 553,
            "ttftMs": 15190
          },
          {
            "line": 3616,
            "rx": "2026-11-01T04:48:55.585Z",
            "sessionId": "84958644-e3f3-4190-983b-d4b0dc2b3d80",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 51571
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-craftbook-powerpoint-deck",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-powerpoint-deck",
        "repetition": 3
      },
      "trialId": "craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-11-01T05-17-19-131Z-jyul",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-11-01T05-17-19-131Z-jyul/recording/chat-events.jsonl",
      "recordingSha256": "85c1afb0e2463c34b05061351788be5e74c1be75c86e734c41e37228fbba790d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-mlx-gemma4-12b-q4-2026-11-01T05-17-19-131Z-jyul/metrics.json",
      "metricsSha256": "21e621b5cd64555828d18e5dd96d5e15937e19a487df0745c9e53b0ae4690f3a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1109,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T05:17:41.053Z",
            "sessionId": "18054787-02ad-4105-8231-63d41e90df6c",
            "source": "first-token-detail",
            "tokensPerSec": 1109,
            "ttftMs": 15404
          },
          {
            "line": 143,
            "rx": "2026-11-01T05:18:11.056Z",
            "sessionId": "660ea5a0-8367-4e90-a401-55cf97f4e1c2",
            "source": "first-token-detail",
            "tokensPerSec": 1477,
            "ttftMs": 5431
          },
          {
            "line": 292,
            "rx": "2026-11-01T05:18:52.422Z",
            "sessionId": "252a9915-80bb-433c-acd8-3a142230e61f",
            "source": "first-token-detail",
            "tokensPerSec": 1372,
            "ttftMs": 6719
          },
          {
            "line": 433,
            "rx": "2026-11-01T05:19:27.044Z",
            "sessionId": "1d904a63-4034-46df-9a47-7fc66c4732ff",
            "source": "first-token-detail",
            "tokensPerSec": 1306,
            "ttftMs": 6421
          },
          {
            "line": 575,
            "rx": "2026-11-01T05:20:12.982Z",
            "sessionId": "021ee8a5-90a6-473e-9b95-3cec73850373",
            "source": "first-token-detail",
            "tokensPerSec": 1199,
            "ttftMs": 11687
          },
          {
            "line": 2621,
            "rx": "2026-11-01T05:27:27.084Z",
            "sessionId": "021ee8a5-90a6-473e-9b95-3cec73850373",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 871
          },
          {
            "line": 3010,
            "rx": "2026-11-01T05:28:34.667Z",
            "sessionId": "021ee8a5-90a6-473e-9b95-3cec73850373",
            "source": "first-token-detail",
            "tokensPerSec": 915,
            "ttftMs": 623
          },
          {
            "line": 4369,
            "rx": "2026-11-01T05:32:54.762Z",
            "sessionId": "021ee8a5-90a6-473e-9b95-3cec73850373",
            "source": "first-token-detail",
            "tokensPerSec": 1040,
            "ttftMs": 23292
          },
          {
            "line": 4588,
            "rx": "2026-11-01T05:34:30.970Z",
            "sessionId": "021ee8a5-90a6-473e-9b95-3cec73850373",
            "source": "first-token-detail",
            "tokensPerSec": 990,
            "ttftMs": 25079
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-wikipedia-research-brief",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "wikipedia-research-brief",
        "repetition": 3
      },
      "trialId": "wikipedia-research-brief-mlx-gemma4-12b-q4-2026-11-01T05-47-58-329Z-iy7j",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-gemma4-12b-q4-2026-11-01T05-47-58-329Z-iy7j/recording/chat-events.jsonl",
      "recordingSha256": "faa604a4883229f9923651d20f65444fba0d533c2bd7401089071e00893a4ab9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-gemma4-12b-q4-2026-11-01T05-47-58-329Z-iy7j/metrics.json",
      "metricsSha256": "99bc97d6108205766ee7933aa38328c794ccc2acbe6a722c8e2b813c6d6edbbf",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1122,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-01T05:48:10.382Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1059,
            "ttftMs": 10359
          },
          {
            "line": 189,
            "rx": "2026-11-01T05:49:05.117Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1405,
            "ttftMs": 5351
          },
          {
            "line": 282,
            "rx": "2026-11-01T05:49:42.705Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1087,
            "ttftMs": 7813
          },
          {
            "line": 542,
            "rx": "2026-11-01T05:51:20.561Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1180,
            "ttftMs": 10350
          },
          {
            "line": 667,
            "rx": "2026-11-01T05:52:19.333Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1038,
            "ttftMs": 13915
          },
          {
            "line": 780,
            "rx": "2026-11-01T05:52:57.007Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1260,
            "ttftMs": 1399
          },
          {
            "line": 1123,
            "rx": "2026-11-01T05:55:20.618Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1046,
            "ttftMs": 19517
          },
          {
            "line": 1249,
            "rx": "2026-11-01T05:56:03.153Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1157,
            "ttftMs": 1801
          },
          {
            "line": 1366,
            "rx": "2026-11-01T05:56:43.049Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 1252,
            "ttftMs": 1500
          },
          {
            "line": 1724,
            "rx": "2026-11-01T05:59:32.127Z",
            "sessionId": "cfde7485-8ab4-4cd3-9793-37fc7d7a96c5",
            "source": "first-token-detail",
            "tokensPerSec": 956,
            "ttftMs": 29871
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-wikipedia-research-brief",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "wikipedia-research-brief",
        "repetition": 3
      },
      "trialId": "wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-11-01T06-03-50-420Z-dvkk",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-11-01T06-03-50-420Z-dvkk/recording/chat-events.jsonl",
      "recordingSha256": "4c6747dab87365ca5dcf56877ea79018704a450a5d921ec645ba62823235ced5",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-mlx-qwen3-8-27b-q4-2026-11-01T06-03-50-420Z-dvkk/metrics.json",
      "metricsSha256": "9d72d66bc6326e801debbf42b2e8694a0a0233e7903eea404dcfeb5b730469b6",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 520,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 25,
            "rx": "2026-11-01T06:04:03.684Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 565,
            "ttftMs": 11531
          },
          {
            "line": 577,
            "rx": "2026-11-01T06:05:44.787Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 657,
            "ttftMs": 7816
          },
          {
            "line": 673,
            "rx": "2026-11-01T06:06:19.709Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 642,
            "ttftMs": 7617
          },
          {
            "line": 1208,
            "rx": "2026-11-01T06:08:11.531Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 496,
            "ttftMs": 14077
          },
          {
            "line": 1256,
            "rx": "2026-11-01T06:08:51.127Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 485,
            "ttftMs": 18551
          },
          {
            "line": 1348,
            "rx": "2026-11-01T06:09:49.773Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 505,
            "ttftMs": 22009
          },
          {
            "line": 1741,
            "rx": "2026-11-01T06:12:14.016Z",
            "sessionId": "179a18b7-42a6-4323-bf0d-99339c38db0d",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 25771
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-craftbook-week-plan",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-week-plan",
        "repetition": 3
      },
      "trialId": "craftbook-week-plan-mlx-gemma4-12b-q4-2026-11-01T06-14-42-490Z-tv6q",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-gemma4-12b-q4-2026-11-01T06-14-42-490Z-tv6q/recording/chat-events.jsonl",
      "recordingSha256": "b7f50f74c041850d48a1d8b8a9acb7704af15d71c12b5ffa59ab92166f2db0a6",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-gemma4-12b-q4-2026-11-01T06-14-42-490Z-tv6q/metrics.json",
      "metricsSha256": "70a1243b47d6fc8c37875da3996bb2144b74177aa5f4bbbe6a137fe487f2e87e",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1252,
        "sampleCount": 31,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 31,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T06:15:06.583Z",
            "sessionId": "4246ac97-c4df-4f7e-8224-e691fef93c45",
            "source": "first-token-detail",
            "tokensPerSec": 1110,
            "ttftMs": 17542
          },
          {
            "line": 173,
            "rx": "2026-11-01T06:15:45.860Z",
            "sessionId": "4246ac97-c4df-4f7e-8224-e691fef93c45",
            "source": "first-token-detail",
            "tokensPerSec": 1122,
            "ttftMs": 11795
          },
          {
            "line": 568,
            "rx": "2026-11-01T06:16:59.513Z",
            "sessionId": "60aa0112-4555-401b-8baa-63d8751aae65",
            "source": "first-token-detail",
            "tokensPerSec": 1505,
            "ttftMs": 5387
          },
          {
            "line": 766,
            "rx": "2026-11-01T06:17:51.331Z",
            "sessionId": "d252838a-1280-4dab-85d3-9c700da11de1",
            "source": "first-token-detail",
            "tokensPerSec": 1207,
            "ttftMs": 7285
          },
          {
            "line": 886,
            "rx": "2026-11-01T06:18:20.215Z",
            "sessionId": "219d68f2-2a35-4deb-8589-a7ea082b7044",
            "source": "first-token-detail",
            "tokensPerSec": 1196,
            "ttftMs": 6108
          },
          {
            "line": 1065,
            "rx": "2026-11-01T06:19:05.121Z",
            "sessionId": "cce5c914-29ab-4fbc-990d-9a3a803e823f",
            "source": "first-token-detail",
            "tokensPerSec": 1252,
            "ttftMs": 6062
          },
          {
            "line": 1274,
            "rx": "2026-11-01T06:19:55.088Z",
            "sessionId": "6329d0c0-f6e6-4362-bae9-be92b31d394a",
            "source": "first-token-detail",
            "tokensPerSec": 1265,
            "ttftMs": 5916
          },
          {
            "line": 1465,
            "rx": "2026-11-01T06:20:40.856Z",
            "sessionId": "523ec296-cf78-4729-b92c-9f8458304b32",
            "source": "first-token-detail",
            "tokensPerSec": 1143,
            "ttftMs": 6745
          },
          {
            "line": 1566,
            "rx": "2026-11-01T06:21:10.932Z",
            "sessionId": "076121a9-963d-4188-9ae0-10eaad719de7",
            "source": "first-token-detail",
            "tokensPerSec": 1161,
            "ttftMs": 6749
          },
          {
            "line": 1896,
            "rx": "2026-11-01T06:22:20.951Z",
            "sessionId": "9d6b30f0-2da0-404f-bd75-e9d4ef971ae4",
            "source": "first-token-detail",
            "tokensPerSec": 1202,
            "ttftMs": 6710
          },
          {
            "line": 2107,
            "rx": "2026-11-01T06:23:17.790Z",
            "sessionId": "9d6b30f0-2da0-404f-bd75-e9d4ef971ae4",
            "source": "first-token-detail",
            "tokensPerSec": 1051,
            "ttftMs": 11891
          },
          {
            "line": 2965,
            "rx": "2026-11-01T06:25:56.249Z",
            "sessionId": "da8921e8-bcc7-4c10-85f6-15c9d9560c77",
            "source": "first-token-detail",
            "tokensPerSec": 1225,
            "ttftMs": 6925
          },
          {
            "line": 3156,
            "rx": "2026-11-01T06:26:51.290Z",
            "sessionId": "f4bf98eb-d965-43bd-a07e-4cb3465c5ade",
            "source": "first-token-detail",
            "tokensPerSec": 1244,
            "ttftMs": 7036
          },
          {
            "line": 3282,
            "rx": "2026-11-01T06:28:19.281Z",
            "sessionId": "9f630f06-09da-4854-86fd-79fc930ed36e",
            "source": "first-token-detail",
            "tokensPerSec": 1732,
            "ttftMs": 4928
          },
          {
            "line": 3406,
            "rx": "2026-11-01T06:28:43.662Z",
            "sessionId": "9f630f06-09da-4854-86fd-79fc930ed36e",
            "source": "first-token-detail",
            "tokensPerSec": 872,
            "ttftMs": 622
          },
          {
            "line": 3641,
            "rx": "2026-11-01T06:30:34.958Z",
            "sessionId": "3ab760b6-1455-47fa-92b5-e5d48a20ee26",
            "source": "first-token-detail",
            "tokensPerSec": 1597,
            "ttftMs": 5577
          },
          {
            "line": 3885,
            "rx": "2026-11-01T06:32:19.441Z",
            "sessionId": "acfa9050-7b50-4b8c-a9fb-8d09c9fd76e4",
            "source": "first-token-detail",
            "tokensPerSec": 1670,
            "ttftMs": 5043
          },
          {
            "line": 4144,
            "rx": "2026-11-01T06:34:04.027Z",
            "sessionId": "5ce85564-28d0-41ca-b0ed-0cb98aca49a7",
            "source": "first-token-detail",
            "tokensPerSec": 1890,
            "ttftMs": 4614
          },
          {
            "line": 4333,
            "rx": "2026-11-01T06:35:43.900Z",
            "sessionId": "5441be86-7c0b-4c93-ae83-8ebfb1cf2234",
            "source": "first-token-detail",
            "tokensPerSec": 1891,
            "ttftMs": 4424
          },
          {
            "line": 4464,
            "rx": "2026-11-01T06:37:14.118Z",
            "sessionId": "dfaff0a1-5e78-479a-8640-cc679eb65c0f",
            "source": "first-token-detail",
            "tokensPerSec": 1883,
            "ttftMs": 4616
          },
          {
            "line": 4613,
            "rx": "2026-11-01T06:38:49.018Z",
            "sessionId": "b791cc03-eab2-47ca-8758-785145a497b8",
            "source": "first-token-detail",
            "tokensPerSec": 1889,
            "ttftMs": 4434
          },
          {
            "line": 4733,
            "rx": "2026-11-01T06:40:14.284Z",
            "sessionId": "7cb87d9d-baab-41dd-8cde-32a683bee7e6",
            "source": "first-token-detail",
            "tokensPerSec": 1883,
            "ttftMs": 4642
          },
          {
            "line": 4895,
            "rx": "2026-11-01T06:41:49.006Z",
            "sessionId": "24d6cc22-6ba5-4a2d-a767-9ca2b7c15209",
            "source": "first-token-detail",
            "tokensPerSec": 1890,
            "ttftMs": 4339
          },
          {
            "line": 5141,
            "rx": "2026-11-01T06:43:39.160Z",
            "sessionId": "b9299a8f-9ca2-4268-accf-c9101fcef8dd",
            "source": "first-token-detail",
            "tokensPerSec": 1865,
            "ttftMs": 4576
          },
          {
            "line": 5291,
            "rx": "2026-11-01T06:45:09.096Z",
            "sessionId": "98c84ab0-a85c-47d3-86ad-7503d0ca6818",
            "source": "first-token-detail",
            "tokensPerSec": 1910,
            "ttftMs": 4491
          },
          {
            "line": 5382,
            "rx": "2026-11-01T06:45:33.991Z",
            "sessionId": "7130f0f1-5b10-4232-a44c-0edc238476e8",
            "source": "first-token-detail",
            "tokensPerSec": 1573,
            "ttftMs": 9312
          },
          {
            "line": 6011,
            "rx": "2026-11-01T06:47:28.067Z",
            "sessionId": "7130f0f1-5b10-4232-a44c-0edc238476e8",
            "source": "first-token-detail",
            "tokensPerSec": 1198,
            "ttftMs": 7964
          },
          {
            "line": 6082,
            "rx": "2026-11-01T06:47:44.439Z",
            "sessionId": "0d99937a-51f4-40bc-a93f-ca39ca3c5642",
            "source": "first-token-detail",
            "tokensPerSec": 173,
            "ttftMs": 28242
          },
          {
            "line": 6111,
            "rx": "2026-11-01T06:47:58.908Z",
            "sessionId": "0d99937a-51f4-40bc-a93f-ca39ca3c5642",
            "source": "first-token-detail",
            "tokensPerSec": 1454,
            "ttftMs": 3511
          },
          {
            "line": 6143,
            "rx": "2026-11-01T06:48:18.860Z",
            "sessionId": "0d99937a-51f4-40bc-a93f-ca39ca3c5642",
            "source": "first-token-detail",
            "tokensPerSec": 1173,
            "ttftMs": 7769
          },
          {
            "line": 6269,
            "rx": "2026-11-01T06:49:20.559Z",
            "sessionId": "0d99937a-51f4-40bc-a93f-ca39ca3c5642",
            "source": "first-token-detail",
            "tokensPerSec": 1028,
            "ttftMs": 9641
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-craftbook-week-plan",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-week-plan",
        "repetition": 3
      },
      "trialId": "craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-11-01T06-56-53-321Z-3fxg",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-11-01T06-56-53-321Z-3fxg/recording/chat-events.jsonl",
      "recordingSha256": "8601d4def13acb030331acfb2732d4a83d7a05584d54122826208d82a4d038bc",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-mlx-qwen3-8-27b-q4-2026-11-01T06-56-53-321Z-3fxg/metrics.json",
      "metricsSha256": "6f1028d4fae02c504b6e663c4c2ceae5b00a98bb55aedf9db96ad65f62712686",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 555,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-11-01T06:57:25.520Z",
            "sessionId": "24bb19e3-bc50-4197-9165-ac8cddfcd66a",
            "source": "first-token-detail",
            "tokensPerSec": 779,
            "ttftMs": 25530
          },
          {
            "line": 2432,
            "rx": "2026-11-01T07:05:00.134Z",
            "sessionId": "24bb19e3-bc50-4197-9165-ac8cddfcd66a",
            "source": "first-token-detail",
            "tokensPerSec": 450,
            "ttftMs": 32674
          },
          {
            "line": 4507,
            "rx": "2026-11-01T07:11:17.809Z",
            "sessionId": "cbb49a41-31fd-439b-af81-071cdc0d698f",
            "source": "first-token-detail",
            "tokensPerSec": 584,
            "ttftMs": 7479
          },
          {
            "line": 4660,
            "rx": "2026-11-01T07:11:56.271Z",
            "sessionId": "cbb49a41-31fd-439b-af81-071cdc0d698f",
            "source": "first-token-detail",
            "tokensPerSec": 505,
            "ttftMs": 16268
          },
          {
            "line": 4974,
            "rx": "2026-11-01T07:13:33.684Z",
            "sessionId": "24bb19e3-bc50-4197-9165-ac8cddfcd66a",
            "source": "first-token-detail",
            "tokensPerSec": 468,
            "ttftMs": 43402
          },
          {
            "line": 5919,
            "rx": "2026-11-01T07:16:25.210Z",
            "sessionId": "db541be1-2b7b-4181-8086-1623feb86a26",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 14811
          },
          {
            "line": 6600,
            "rx": "2026-11-01T07:18:55.623Z",
            "sessionId": "d93df788-565b-4c15-b2ca-3180f8524e2a",
            "source": "first-token-detail",
            "tokensPerSec": 559,
            "ttftMs": 15144
          },
          {
            "line": 7456,
            "rx": "2026-11-01T07:21:44.250Z",
            "sessionId": "12259100-80ba-451a-820f-588561c6549b",
            "source": "first-token-detail",
            "tokensPerSec": 511,
            "ttftMs": 8761
          },
          {
            "line": 8202,
            "rx": "2026-11-01T07:24:19.068Z",
            "sessionId": "af83c04f-3a7b-43b7-8abc-c1f174f3567a",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 8453
          },
          {
            "line": 10020,
            "rx": "2026-11-01T07:30:00.106Z",
            "sessionId": "0f6277ee-880e-472d-aa4c-21348ac2c296",
            "source": "first-token-detail",
            "tokensPerSec": 555,
            "ttftMs": 9373
          },
          {
            "line": 10354,
            "rx": "2026-11-01T07:31:21.262Z",
            "sessionId": "3b888a17-fc91-41ae-90a9-fe1dc023e0e0",
            "source": "first-token-detail",
            "tokensPerSec": 514,
            "ttftMs": 10531
          },
          {
            "line": 15902,
            "rx": "2026-11-01T07:48:16.672Z",
            "sessionId": "2c3bd597-8423-4e46-a16c-653d7a819c6b",
            "source": "first-token-detail",
            "tokensPerSec": 568,
            "ttftMs": 10557
          },
          {
            "line": 17055,
            "rx": "2026-11-01T07:52:14.766Z",
            "sessionId": "b29224ed-ee35-4ee2-9029-6aacfce97878",
            "source": "first-token-detail",
            "tokensPerSec": 521,
            "ttftMs": 22564
          },
          {
            "line": 18175,
            "rx": "2026-11-01T07:55:27.537Z",
            "sessionId": "7a24e38b-76f2-4dd2-82cc-77452a432368",
            "source": "first-token-detail",
            "tokensPerSec": 289,
            "ttftMs": 151262
          },
          {
            "line": 21169,
            "rx": "2026-11-01T08:05:28.019Z",
            "sessionId": "9bb2f1d8-2a71-49a9-8ab8-fcb0f8039dfd",
            "source": "first-token-detail",
            "tokensPerSec": 797,
            "ttftMs": 11586
          },
          {
            "line": 21313,
            "rx": "2026-11-01T08:06:13.547Z",
            "sessionId": "c483d1a8-c8e8-446c-a647-27cf45bfc5ee",
            "source": "first-token-detail",
            "tokensPerSec": 720,
            "ttftMs": 25654
          },
          {
            "line": 21585,
            "rx": "2026-11-01T08:07:11.981Z",
            "sessionId": "c681a4b6-fe0e-4dde-99fe-8bc708cce5ce",
            "source": "first-token-detail",
            "tokensPerSec": 592,
            "ttftMs": 6667
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-craftbook-ab-test-readout",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-ab-test-readout",
        "repetition": 3
      },
      "trialId": "craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-11-01T10-50-56-405Z-bnqy",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-11-01T10-50-56-405Z-bnqy/recording/chat-events.jsonl",
      "recordingSha256": "980841749488a76ab6cd8fb97beec3d8c2bec87681a15f1597fc411a7dc145b7",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-qwen3-8-27b-q4-2026-11-01T10-50-56-405Z-bnqy/metrics.json",
      "metricsSha256": "d28f05aab67e68ff5df4eca5b38a2274dbc4380e083dd93cf09248ee1c8dda01",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 626,
        "sampleCount": 10,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 10,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-11-01T10:51:25.234Z",
            "sessionId": "841e423b-9b5a-4183-ac1d-e8cd42436be9",
            "source": "first-token-detail",
            "tokensPerSec": 812,
            "ttftMs": 22361
          },
          {
            "line": 375,
            "rx": "2026-11-01T10:52:42.458Z",
            "sessionId": "841e423b-9b5a-4183-ac1d-e8cd42436be9",
            "source": "first-token-detail",
            "tokensPerSec": 656,
            "ttftMs": 24418
          },
          {
            "line": 1141,
            "rx": "2026-11-01T10:55:02.322Z",
            "sessionId": "337ee81f-5f1b-480a-b3d9-1ecab1fadd03",
            "source": "first-token-detail",
            "tokensPerSec": 596,
            "ttftMs": 14249
          },
          {
            "line": 1648,
            "rx": "2026-11-01T10:56:51.721Z",
            "sessionId": "8f383e80-aa18-498a-bf66-18a3a4befe06",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 18666
          },
          {
            "line": 4045,
            "rx": "2026-11-01T11:04:07.617Z",
            "sessionId": "93573268-a262-45c3-aad5-f835ed7647c1",
            "source": "first-token-detail",
            "tokensPerSec": 556,
            "ttftMs": 9522
          },
          {
            "line": 4593,
            "rx": "2026-11-01T11:06:49.566Z",
            "sessionId": "60992192-49e7-4e82-805b-735d22f9d2ba",
            "source": "first-token-detail",
            "tokensPerSec": 885,
            "ttftMs": 6286
          },
          {
            "line": 6542,
            "rx": "2026-11-01T11:13:13.233Z",
            "sessionId": "2e9a22d2-5141-44eb-a298-7dc1efb59d9c",
            "source": "first-token-detail",
            "tokensPerSec": 548,
            "ttftMs": 9815
          },
          {
            "line": 6846,
            "rx": "2026-11-01T11:15:09.840Z",
            "sessionId": "8977666a-6a76-4a8a-9639-26dfd3ff9420",
            "source": "first-token-detail",
            "tokensPerSec": 886,
            "ttftMs": 6381
          },
          {
            "line": 8882,
            "rx": "2026-11-01T11:21:53.250Z",
            "sessionId": "437b39c9-6fa0-4570-b103-0427ee8539ca",
            "source": "first-token-detail",
            "tokensPerSec": 584,
            "ttftMs": 14676
          },
          {
            "line": 9067,
            "rx": "2026-11-01T11:22:33.619Z",
            "sessionId": "4e1ec2f6-5eb8-46a7-a272-2af5859a1c1d",
            "source": "first-token-detail",
            "tokensPerSec": 720,
            "ttftMs": 23304
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-craftbook-ab-test-readout",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-ab-test-readout",
        "repetition": 3
      },
      "trialId": "craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-11-01T11-48-40-583Z-yu9s",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-11-01T11-48-40-583Z-yu9s/recording/chat-events.jsonl",
      "recordingSha256": "f88dfb8eb8905527d4d7b7f88c974ac0cb4137cbe7226aca0762772ce41cbf42",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-mlx-gemma4-12b-q4-2026-11-01T11-48-40-583Z-yu9s/metrics.json",
      "metricsSha256": "28d88a4b32be9332f7384be65ce766c23d254db86c6cb85ad65a8106cd1b95b9",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1175,
        "sampleCount": 19,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 19,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T11:49:03.609Z",
            "sessionId": "106f6c85-6da7-45ae-a3e0-073f0ba93ee8",
            "source": "first-token-detail",
            "tokensPerSec": 1109,
            "ttftMs": 16518
          },
          {
            "line": 139,
            "rx": "2026-11-01T11:49:34.326Z",
            "sessionId": "106f6c85-6da7-45ae-a3e0-073f0ba93ee8",
            "source": "first-token-detail",
            "tokensPerSec": 1180,
            "ttftMs": 12193
          },
          {
            "line": 884,
            "rx": "2026-11-01T11:51:39.514Z",
            "sessionId": "68b7d2f1-3747-41e8-9842-95fb203ab106",
            "source": "first-token-detail",
            "tokensPerSec": 1175,
            "ttftMs": 7327
          },
          {
            "line": 994,
            "rx": "2026-11-01T11:52:04.685Z",
            "sessionId": "68b7d2f1-3747-41e8-9842-95fb203ab106",
            "source": "first-token-detail",
            "tokensPerSec": 592,
            "ttftMs": 850
          },
          {
            "line": 1370,
            "rx": "2026-11-01T11:53:20.548Z",
            "sessionId": "533cba63-1f15-42d6-a787-45caea684b3f",
            "source": "first-token-detail",
            "tokensPerSec": 1133,
            "ttftMs": 8316
          },
          {
            "line": 1542,
            "rx": "2026-11-01T11:53:58.539Z",
            "sessionId": "6eaa278d-7a3f-456b-a2a6-de64bdfa96ea",
            "source": "first-token-detail",
            "tokensPerSec": 1229,
            "ttftMs": 6400
          },
          {
            "line": 1999,
            "rx": "2026-11-01T11:55:19.259Z",
            "sessionId": "6eaa278d-7a3f-456b-a2a6-de64bdfa96ea",
            "source": "first-token-detail",
            "tokensPerSec": 640,
            "ttftMs": 838
          },
          {
            "line": 2118,
            "rx": "2026-11-01T11:55:50.172Z",
            "sessionId": "c0c2aa08-8aba-42e5-b833-0cb3ba07b71a",
            "source": "first-token-detail",
            "tokensPerSec": 1052,
            "ttftMs": 7959
          },
          {
            "line": 2265,
            "rx": "2026-11-01T11:56:24.344Z",
            "sessionId": "30a97554-9062-4db2-92c8-28d148ea20c2",
            "source": "first-token-detail",
            "tokensPerSec": 1165,
            "ttftMs": 7061
          },
          {
            "line": 2376,
            "rx": "2026-11-01T11:56:46.285Z",
            "sessionId": "30a97554-9062-4db2-92c8-28d148ea20c2",
            "source": "first-token-detail",
            "tokensPerSec": 599,
            "ttftMs": 961
          },
          {
            "line": 4122,
            "rx": "2026-11-01T12:01:36.209Z",
            "sessionId": "30a97554-9062-4db2-92c8-28d148ea20c2",
            "source": "first-token-detail",
            "tokensPerSec": 479,
            "ttftMs": 614
          },
          {
            "line": 4570,
            "rx": "2026-11-01T12:03:19.190Z",
            "sessionId": "4caf14b0-99e1-41b9-8ac3-386f58ba26db",
            "source": "first-token-detail",
            "tokensPerSec": 1280,
            "ttftMs": 6873
          },
          {
            "line": 8007,
            "rx": "2026-11-01T12:12:59.218Z",
            "sessionId": "025b4a5e-ab54-4c5e-84d0-959dbd873cf2",
            "source": "first-token-detail",
            "tokensPerSec": 1316,
            "ttftMs": 6649
          },
          {
            "line": 8219,
            "rx": "2026-11-01T12:14:42.489Z",
            "sessionId": "3c6d6926-ec4d-41e9-80c6-b22dd2cca111",
            "source": "first-token-detail",
            "tokensPerSec": 1867,
            "ttftMs": 4828
          },
          {
            "line": 9855,
            "rx": "2026-11-01T12:20:26.328Z",
            "sessionId": "3c6d6926-ec4d-41e9-80c6-b22dd2cca111",
            "source": "first-token-detail",
            "tokensPerSec": 1114,
            "ttftMs": 18542
          },
          {
            "line": 9989,
            "rx": "2026-11-01T12:21:52.816Z",
            "sessionId": "4692ab99-f5e1-418f-81ba-e8ebd2024870",
            "source": "first-token-detail",
            "tokensPerSec": 1917,
            "ttftMs": 4989
          },
          {
            "line": 10161,
            "rx": "2026-11-01T12:22:27.813Z",
            "sessionId": "49c6a23f-ee83-4674-b780-791f9825f25b",
            "source": "first-token-detail",
            "tokensPerSec": 1757,
            "ttftMs": 6323
          },
          {
            "line": 10393,
            "rx": "2026-11-01T12:23:11.543Z",
            "sessionId": "91004edc-ac03-4faa-b7da-208e5750efba",
            "source": "first-token-detail",
            "tokensPerSec": 1382,
            "ttftMs": 3620
          },
          {
            "line": 10418,
            "rx": "2026-11-01T12:23:24.485Z",
            "sessionId": "91004edc-ac03-4faa-b7da-208e5750efba",
            "source": "first-token-detail",
            "tokensPerSec": 1524,
            "ttftMs": 3066
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-craftbook-annotated-bibliography",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-annotated-bibliography",
        "repetition": 3
      },
      "trialId": "craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-11-01T12-24-06-811Z-d41h",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-11-01T12-24-06-811Z-d41h/recording/chat-events.jsonl",
      "recordingSha256": "d4c40a7d30656e9a10033c77b5ffd24d34ffae584fd835e8f7adeb361887e4fb",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-qwen3-8-27b-q4-2026-11-01T12-24-06-811Z-d41h/metrics.json",
      "metricsSha256": "573eb5c881b63ab0937406ff81fcbe3fe78b5732786a0dc23e1f34eeecb3b769",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 621,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T12:24:32.458Z",
            "sessionId": "4f31fb89-463f-4c0d-bac0-6ab46fd30cd9",
            "source": "first-token-detail",
            "tokensPerSec": 638,
            "ttftMs": 19140
          },
          {
            "line": 186,
            "rx": "2026-11-01T12:25:16.462Z",
            "sessionId": "c74dd4ff-3abc-4450-a6a7-1103bafacc27",
            "source": "first-token-detail",
            "tokensPerSec": 680,
            "ttftMs": 18094
          },
          {
            "line": 689,
            "rx": "2026-11-01T12:26:56.997Z",
            "sessionId": "decceeff-6b22-46ec-9b77-5e458a165ffd",
            "source": "first-token-detail",
            "tokensPerSec": 604,
            "ttftMs": 13577
          },
          {
            "line": 1046,
            "rx": "2026-11-01T12:28:18.926Z",
            "sessionId": "4214fe83-1443-417e-822a-d47f748bd7e1",
            "source": "first-token-detail",
            "tokensPerSec": 546,
            "ttftMs": 15464
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-craftbook-annotated-bibliography",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-annotated-bibliography",
        "repetition": 3
      },
      "trialId": "craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-11-01T12-37-17-075Z-ced9",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-11-01T12-37-17-075Z-ced9/recording/chat-events.jsonl",
      "recordingSha256": "4db42f50d3fa335a588c8d5f994a06704ce4e7758d79a1b6e9b110435a44977c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-mlx-gemma4-12b-q4-2026-11-01T12-37-17-075Z-ced9/metrics.json",
      "metricsSha256": "6d38a25606695a405acf2c10f64542376b313334ff71d6e8a46da19c50f52e99",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1276,
        "sampleCount": 8,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 8,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T12:37:37.287Z",
            "sessionId": "ee548e84-bbfc-47c3-91d9-389359ed2d56",
            "source": "first-token-detail",
            "tokensPerSec": 1115,
            "ttftMs": 13697
          },
          {
            "line": 280,
            "rx": "2026-11-01T12:38:27.479Z",
            "sessionId": "daa14151-206c-4de0-b7b3-274f0da61269",
            "source": "first-token-detail",
            "tokensPerSec": 1222,
            "ttftMs": 8831
          },
          {
            "line": 531,
            "rx": "2026-11-01T12:39:35.163Z",
            "sessionId": "1b3927ba-b2c1-4afb-b217-3d19f4048e7c",
            "source": "first-token-detail",
            "tokensPerSec": 1372,
            "ttftMs": 6449
          },
          {
            "line": 725,
            "rx": "2026-11-01T12:40:25.197Z",
            "sessionId": "d2b4b78a-3fe4-49bc-9e44-cbf53aca1b70",
            "source": "first-token-detail",
            "tokensPerSec": 1307,
            "ttftMs": 6564
          },
          {
            "line": 903,
            "rx": "2026-11-01T12:41:11.053Z",
            "sessionId": "bad640f1-1467-4ec0-94bd-af435545e374",
            "source": "first-token-detail",
            "tokensPerSec": 1248,
            "ttftMs": 7345
          },
          {
            "line": 1045,
            "rx": "2026-11-01T12:41:54.257Z",
            "sessionId": "da2cb7e3-7bec-4e85-90d3-59f6092cdd2d",
            "source": "first-token-detail",
            "tokensPerSec": 1343,
            "ttftMs": 5641
          },
          {
            "line": 2682,
            "rx": "2026-11-01T12:46:45.383Z",
            "sessionId": "d12ba435-6d14-4283-b3e1-58fce902dcd3",
            "source": "first-token-detail",
            "tokensPerSec": 1304,
            "ttftMs": 6517
          },
          {
            "line": 2814,
            "rx": "2026-11-01T12:47:25.708Z",
            "sessionId": "e20f2aeb-4113-4a65-af39-e399ab50e246",
            "source": "first-token-detail",
            "tokensPerSec": 1107,
            "ttftMs": 6986
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-productivity-craftbook-research-to-document",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "productivity",
        "scenario": "craftbook-research-to-document",
        "repetition": 3
      },
      "trialId": "craftbook-research-to-document-mlx-gemma4-12b-q4-2026-11-01T13-02-27-242Z-hrqi",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-gemma4-12b-q4-2026-11-01T13-02-27-242Z-hrqi/recording/chat-events.jsonl",
      "recordingSha256": "8674eae9eb24153e48af7087b557b6e4578fa9dde33c26dd7eb7ce336617d076",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-gemma4-12b-q4-2026-11-01T13-02-27-242Z-hrqi/metrics.json",
      "metricsSha256": "11e26bc4bfb7423798843cfcd63ea8da2729f9415303b49c7e016b214473decf",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1204.5,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T13:02:47.537Z",
            "sessionId": "200ebab4-f16f-482f-b14d-c3fc10655cc6",
            "source": "first-token-detail",
            "tokensPerSec": 1108,
            "ttftMs": 13733
          },
          {
            "line": 109,
            "rx": "2026-11-01T13:03:10.719Z",
            "sessionId": "e474dde9-e591-462f-acc6-8f489e4291a1",
            "source": "first-token-detail",
            "tokensPerSec": 1309,
            "ttftMs": 6954
          },
          {
            "line": 199,
            "rx": "2026-11-01T13:03:35.663Z",
            "sessionId": "be6243ab-1319-4ff6-bc16-f9b643643a84",
            "source": "first-token-detail",
            "tokensPerSec": 1325,
            "ttftMs": 6920
          },
          {
            "line": 356,
            "rx": "2026-11-01T13:04:27.018Z",
            "sessionId": "be6243ab-1319-4ff6-bc16-f9b643643a84",
            "source": "first-token-detail",
            "tokensPerSec": 1227,
            "ttftMs": 12553
          },
          {
            "line": 2393,
            "rx": "2026-11-01T13:11:27.876Z",
            "sessionId": "be6243ab-1319-4ff6-bc16-f9b643643a84",
            "source": "first-token-detail",
            "tokensPerSec": 1013,
            "ttftMs": 20396
          },
          {
            "line": 2734,
            "rx": "2026-11-01T13:14:15.972Z",
            "sessionId": "0e8e4c76-56d5-4549-bd8a-f5a766b1c536",
            "source": "first-token-detail",
            "tokensPerSec": 1182,
            "ttftMs": 11351
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-productivity-craftbook-research-to-document",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "productivity",
        "scenario": "craftbook-research-to-document",
        "repetition": 3
      },
      "trialId": "craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-11-01T13-19-07-756Z-nrew",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-11-01T13-19-07-756Z-nrew/recording/chat-events.jsonl",
      "recordingSha256": "d3cd6806244fa83daef71cd2f7e0eb03ed794be6cbd25f475224b6fecfc133c3",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-mlx-qwen3-8-27b-q4-2026-11-01T13-19-07-756Z-nrew/metrics.json",
      "metricsSha256": "e53a4d1150eeb1a1198b87963487bcad910fc1b447dfa1bc6a34128756ca2fec",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 555,
        "sampleCount": 5,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 5,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-11-01T13:19:34.519Z",
            "sessionId": "9de27dac-0e1a-4779-9ff8-703d40279e37",
            "source": "first-token-detail",
            "tokensPerSec": 577,
            "ttftMs": 20251
          },
          {
            "line": 187,
            "rx": "2026-11-01T13:20:14.068Z",
            "sessionId": "5cce48a9-9828-43e1-ad17-dc9d5e1f98cf",
            "source": "first-token-detail",
            "tokensPerSec": 617,
            "ttftMs": 14784
          },
          {
            "line": 764,
            "rx": "2026-11-01T13:22:04.670Z",
            "sessionId": "ca517ed9-7b5d-4f35-9b77-99447a4ea3f7",
            "source": "first-token-detail",
            "tokensPerSec": 555,
            "ttftMs": 15294
          },
          {
            "line": 1021,
            "rx": "2026-11-01T13:23:38.036Z",
            "sessionId": "ca517ed9-7b5d-4f35-9b77-99447a4ea3f7",
            "source": "first-token-detail",
            "tokensPerSec": 503,
            "ttftMs": 53119
          },
          {
            "line": 1295,
            "rx": "2026-11-01T13:29:00.392Z",
            "sessionId": "575c0c33-3179-441a-b692-8dc33caab6ea",
            "source": "first-token-detail",
            "tokensPerSec": 500,
            "ttftMs": 50261
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-fix-squisq-bugs",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "fix-squisq-bugs",
        "repetition": 3
      },
      "trialId": "fix-squisq-bugs-mlx-gemma4-12b-q4-2026-11-01T13-32-19-217Z-9ht8",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-gemma4-12b-q4-2026-11-01T13-32-19-217Z-9ht8/recording/chat-events.jsonl",
      "recordingSha256": "7d47ce0393461e7c1ee951c8cd842673d4423514821eb65e13cbc9987541222e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-gemma4-12b-q4-2026-11-01T13-32-19-217Z-9ht8/metrics.json",
      "metricsSha256": "e9eddcefc607a11e48cb5dc5e26e535efa3e0fdf6b786eeceaf95c54341acc26",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1002,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T13:32:30.726Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 1170,
            "ttftMs": 9873
          },
          {
            "line": 598,
            "rx": "2026-11-01T13:35:03.189Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 1098,
            "ttftMs": 14618
          },
          {
            "line": 644,
            "rx": "2026-11-01T13:35:47.218Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 1140,
            "ttftMs": 15880
          },
          {
            "line": 701,
            "rx": "2026-11-01T13:36:38.012Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 1002,
            "ttftMs": 21481
          },
          {
            "line": 969,
            "rx": "2026-11-01T13:39:15.516Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 1001,
            "ttftMs": 28029
          },
          {
            "line": 1132,
            "rx": "2026-11-01T13:40:32.922Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 963,
            "ttftMs": 30154
          },
          {
            "line": 1302,
            "rx": "2026-11-01T13:41:55.335Z",
            "sessionId": "2aa49559-b32c-4da8-a605-d30a5f292414",
            "source": "first-token-detail",
            "tokensPerSec": 963,
            "ttftMs": 32780
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-fix-squisq-bugs",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "fix-squisq-bugs",
        "repetition": 3
      },
      "trialId": "fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-11-01T13-46-11-565Z-xs5u",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-11-01T13-46-11-565Z-xs5u/recording/chat-events.jsonl",
      "recordingSha256": "be59aa4ed0aa2571dcce9bf97f4e4501cd5045a56d59e2e061ae2f335cab6fae",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-mlx-qwen3-8-27b-q4-2026-11-01T13-46-11-565Z-xs5u/metrics.json",
      "metricsSha256": "c3eeacfda74f2e8c91f7101887f0cc062f4af02e613da3b8f575ad3875aeb666",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 608,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T13:46:26.138Z",
            "sessionId": "f5fda7d9-2cca-49fb-a148-74d0a10d2146",
            "source": "first-token-detail",
            "tokensPerSec": 608,
            "ttftMs": 12916
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-dev-craftbook-routing",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "dev-craftbook-routing",
        "repetition": 3
      },
      "trialId": "dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-11-01T13-58-35-221Z-tw5v",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-11-01T13-58-35-221Z-tw5v/recording/chat-events.jsonl",
      "recordingSha256": "53933f91e884e4a96254b7b5a92b789287da5fed373ca82ce9c825e01fa6c509",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-qwen3-8-27b-q4-2026-11-01T13-58-35-221Z-tw5v/metrics.json",
      "metricsSha256": "7015d598c7b52de8adc842636aeb936d4af554b2c8385a38dd1cb620597e80de",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 676.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T13:59:05.674Z",
            "sessionId": "0d0c8a0e-9b79-404c-afb7-d33c2a4ed728",
            "source": "first-token-detail",
            "tokensPerSec": 521,
            "ttftMs": 28747
          },
          {
            "line": 2561,
            "rx": "2026-11-01T14:14:50.673Z",
            "sessionId": "f1997387-70a8-4d56-bcc1-c8524d87e421",
            "source": "first-token-detail",
            "tokensPerSec": 832,
            "ttftMs": 20628
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-dev-craftbook-routing",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "dev-craftbook-routing",
        "repetition": 3
      },
      "trialId": "dev-craftbook-routing-mlx-gemma4-12b-q4-2026-11-01T14-35-50-333Z-ec97",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-gemma4-12b-q4-2026-11-01T14-35-50-333Z-ec97/recording/chat-events.jsonl",
      "recordingSha256": "5518e4acf3f6cd834f2afdef5a9a0bede5582193967b995044a6cd295bd11359",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-mlx-gemma4-12b-q4-2026-11-01T14-35-50-333Z-ec97/metrics.json",
      "metricsSha256": "820f2f65128df07d9cc95d28fc1f6c3ad3c430f365590fdd7a9f5fff1d5e14d4",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 558,
        "sampleCount": 11,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 11,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-11-01T14:36:07.218Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 1122,
            "ttftMs": 15274
          },
          {
            "line": 1473,
            "rx": "2026-11-01T14:40:06.879Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 364,
            "ttftMs": 440
          },
          {
            "line": 1620,
            "rx": "2026-11-01T14:40:28.824Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 459,
            "ttftMs": 492
          },
          {
            "line": 1951,
            "rx": "2026-11-01T14:41:28.024Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 550,
            "ttftMs": 716
          },
          {
            "line": 2076,
            "rx": "2026-11-01T14:41:50.892Z",
            "sessionId": "68d0feed-a18a-4f71-85d9-fc22fb22719c",
            "source": "first-token-detail",
            "tokensPerSec": 1053,
            "ttftMs": 6177
          },
          {
            "line": 2116,
            "rx": "2026-11-01T14:41:56.587Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 588
          },
          {
            "line": 2157,
            "rx": "2026-11-01T14:42:04.506Z",
            "sessionId": "68d0feed-a18a-4f71-85d9-fc22fb22719c",
            "source": "first-token-detail",
            "tokensPerSec": 999,
            "ttftMs": 4566
          },
          {
            "line": 2362,
            "rx": "2026-11-01T14:42:44.094Z",
            "sessionId": "68d0feed-a18a-4f71-85d9-fc22fb22719c",
            "source": "first-token-detail",
            "tokensPerSec": 979,
            "ttftMs": 6636
          },
          {
            "line": 2394,
            "rx": "2026-11-01T14:42:52.098Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 358,
            "ttftMs": 507
          },
          {
            "line": 2845,
            "rx": "2026-11-01T14:44:00.593Z",
            "sessionId": "68d0feed-a18a-4f71-85d9-fc22fb22719c",
            "source": "first-token-detail",
            "tokensPerSec": 691,
            "ttftMs": 1270
          },
          {
            "line": 2878,
            "rx": "2026-11-01T14:44:08.725Z",
            "sessionId": "28539f74-4bec-4a5a-a597-dd14fd004431",
            "source": "first-token-detail",
            "tokensPerSec": 518,
            "ttftMs": 635
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-interface-contract",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "interface-contract",
        "repetition": 3
      },
      "trialId": "interface-contract-mlx-qwen3-8-27b-q4-2026-11-01T14-44-34-546Z-rt04",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-qwen3-8-27b-q4-2026-11-01T14-44-34-546Z-rt04/recording/chat-events.jsonl",
      "recordingSha256": "7842bd6b10cfc857810f1543acccddcf24e9d88bfb89b55657a303890a70a121",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-qwen3-8-27b-q4-2026-11-01T14-44-34-546Z-rt04/metrics.json",
      "metricsSha256": "6c9d87ae29c949e3baeceb93d03a0328770ce7cc3e6eef1a7f056d1957aad335",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 581,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T14:45:03.667Z",
            "sessionId": "e3cf5021-8a70-4727-a82d-d10631dad22b",
            "source": "first-token-detail",
            "tokensPerSec": 562,
            "ttftMs": 27420
          },
          {
            "line": 274,
            "rx": "2026-11-01T14:45:51.746Z",
            "sessionId": "61643cc9-97da-4de2-ac77-2cae344feea8",
            "source": "first-token-detail",
            "tokensPerSec": 600,
            "ttftMs": 10561
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-interface-contract",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "interface-contract",
        "repetition": 3
      },
      "trialId": "interface-contract-mlx-gemma4-12b-q4-2026-11-01T14-49-42-350Z-poi8",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-gemma4-12b-q4-2026-11-01T14-49-42-350Z-poi8/recording/chat-events.jsonl",
      "recordingSha256": "6cc7f8fc28febf11591953857a678ce52757f5a7802a728f820866f61162ab50",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-interface-contract/interface-contract/interface-contract-mlx-gemma4-12b-q4-2026-11-01T14-49-42-350Z-poi8/metrics.json",
      "metricsSha256": "3458a7f6328c6d7ecca73302295f0eb32c862d60b912a292896df2e451f4e6cc",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1300.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 24,
            "rx": "2026-11-01T14:49:59.310Z",
            "sessionId": "0d907c22-90f5-4551-811b-315f994664e9",
            "source": "first-token-detail",
            "tokensPerSec": 1144,
            "ttftMs": 15277
          },
          {
            "line": 197,
            "rx": "2026-11-01T14:50:33.971Z",
            "sessionId": "eb534f4c-3f15-4e42-bbe7-339e104b7c63",
            "source": "first-token-detail",
            "tokensPerSec": 1457,
            "ttftMs": 5089
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-craftbook-code-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-code-review",
        "repetition": 3
      },
      "trialId": "craftbook-code-review-mlx-gemma4-12b-q4-2026-11-01T14-56-53-006Z-qxih",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-gemma4-12b-q4-2026-11-01T14-56-53-006Z-qxih/recording/chat-events.jsonl",
      "recordingSha256": "eaa8a0f476e1e7626476a538b5a2cc379a325d6cdcc0ff4b091a03b9ff6428ea",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-gemma4-12b-q4-2026-11-01T14-56-53-006Z-qxih/metrics.json",
      "metricsSha256": "22bec09b388d16feb92d0b09ee34bac2314fba8e866c2ba7cf9719f27ce6891b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1239,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T14:57:12.899Z",
            "sessionId": "c2516f16-2e35-4ba5-a94c-35ec944eaa3d",
            "source": "first-token-detail",
            "tokensPerSec": 1082,
            "ttftMs": 13326
          },
          {
            "line": 109,
            "rx": "2026-11-01T14:57:39.017Z",
            "sessionId": "c2516f16-2e35-4ba5-a94c-35ec944eaa3d",
            "source": "first-token-detail",
            "tokensPerSec": 1130,
            "ttftMs": 9515
          },
          {
            "line": 568,
            "rx": "2026-11-01T14:59:06.482Z",
            "sessionId": "c2516f16-2e35-4ba5-a94c-35ec944eaa3d",
            "source": "first-token-detail",
            "tokensPerSec": 1348,
            "ttftMs": 6901
          },
          {
            "line": 629,
            "rx": "2026-11-01T14:59:24.389Z",
            "sessionId": "b80c4caf-63a7-4767-b69d-23d365e8fd69",
            "source": "first-token-detail",
            "tokensPerSec": 1555,
            "ttftMs": 4167
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-craftbook-code-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-code-review",
        "repetition": 3
      },
      "trialId": "craftbook-code-review-mlx-qwen3-8-27b-q4-2026-11-01T15-19-55-409Z-5gkp",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-qwen3-8-27b-q4-2026-11-01T15-19-55-409Z-5gkp/recording/chat-events.jsonl",
      "recordingSha256": "cfbdab1b7d73cc60e94f2f62159e37a30093d4818df038a94d107251ed988b2e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-mlx-qwen3-8-27b-q4-2026-11-01T15-19-55-409Z-5gkp/metrics.json",
      "metricsSha256": "c78d27cf33c37e5e5eb87ce7a7e0b64d04ef54978303d596177e6999d7b52678",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 681.5,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T15:20:16.470Z",
            "sessionId": "82ed58c7-d94d-48d3-8836-aeb666901edd",
            "source": "first-token-detail",
            "tokensPerSec": 895,
            "ttftMs": 14571
          },
          {
            "line": 382,
            "rx": "2026-11-01T15:21:28.643Z",
            "sessionId": "82ed58c7-d94d-48d3-8836-aeb666901edd",
            "source": "first-token-detail",
            "tokensPerSec": 660,
            "ttftMs": 16514
          },
          {
            "line": 797,
            "rx": "2026-11-01T15:22:46.896Z",
            "sessionId": "82ed58c7-d94d-48d3-8836-aeb666901edd",
            "source": "first-token-detail",
            "tokensPerSec": 661,
            "ttftMs": 9770
          },
          {
            "line": 882,
            "rx": "2026-11-01T15:23:09.678Z",
            "sessionId": "55c73b46-2b78-4f6e-b5ec-5a92277fc2b6",
            "source": "first-token-detail",
            "tokensPerSec": 702,
            "ttftMs": 6923
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-craftbook-api-contract-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-api-contract-review",
        "repetition": 3
      },
      "trialId": "craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-11-01T15-23-38-905Z-ftx0",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-11-01T15-23-38-905Z-ftx0/recording/chat-events.jsonl",
      "recordingSha256": "dcaa476c16e496a64749c612527eb0126c0939c29e94e16b1b53f920bc36d2d9",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-gemma4-12b-q4-2026-11-01T15-23-38-905Z-ftx0/metrics.json",
      "metricsSha256": "9525f38fd17cf7f0795ab798c4fa9f6797d0e87a2905bac8c56d270c72860848",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1099,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T15:23:59.559Z",
            "sessionId": "54e62df7-5f76-4751-8f77-fddadd2fb21d",
            "source": "first-token-detail",
            "tokensPerSec": 1125,
            "ttftMs": 14140
          },
          {
            "line": 349,
            "rx": "2026-11-01T15:25:18.067Z",
            "sessionId": "a6783928-5540-4e14-9dd9-eeda5faed773",
            "source": "first-token-detail",
            "tokensPerSec": 1099,
            "ttftMs": 7525
          },
          {
            "line": 532,
            "rx": "2026-11-01T15:26:10.884Z",
            "sessionId": "18f389cc-d384-4dc9-aaea-b0b679cdac4a",
            "source": "first-token-detail",
            "tokensPerSec": 1282,
            "ttftMs": 5431
          },
          {
            "line": 699,
            "rx": "2026-11-01T15:26:58.675Z",
            "sessionId": "18f389cc-d384-4dc9-aaea-b0b679cdac4a",
            "source": "first-token-detail",
            "tokensPerSec": 1195,
            "ttftMs": 8139
          },
          {
            "line": 1015,
            "rx": "2026-11-01T15:28:00.640Z",
            "sessionId": "18f389cc-d384-4dc9-aaea-b0b679cdac4a",
            "source": "first-token-detail",
            "tokensPerSec": 1042,
            "ttftMs": 10077
          },
          {
            "line": 1138,
            "rx": "2026-11-01T15:28:31.622Z",
            "sessionId": "18f389cc-d384-4dc9-aaea-b0b679cdac4a",
            "source": "first-token-detail",
            "tokensPerSec": 1035,
            "ttftMs": 11158
          },
          {
            "line": 1737,
            "rx": "2026-11-01T15:30:07.907Z",
            "sessionId": "18f389cc-d384-4dc9-aaea-b0b679cdac4a",
            "source": "first-token-detail",
            "tokensPerSec": 799,
            "ttftMs": 789
          },
          {
            "line": 1865,
            "rx": "2026-11-01T15:30:42.731Z",
            "sessionId": "c2df800b-86c0-4826-a45d-df1337056ed0",
            "source": "first-token-detail",
            "tokensPerSec": 1078,
            "ttftMs": 7227
          },
          {
            "line": 1983,
            "rx": "2026-11-01T15:31:06.546Z",
            "sessionId": "360f5ffa-ab05-4881-9b8c-92e3492e8d49",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 4141
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-craftbook-api-contract-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-api-contract-review",
        "repetition": 3
      },
      "trialId": "craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-11-01T15-39-24-739Z-zzmy",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-11-01T15-39-24-739Z-zzmy/recording/chat-events.jsonl",
      "recordingSha256": "c46b89b1b875fb806e902174aaaf7e14d4b103e490f31f17ec00a4ca3181a255",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-mlx-qwen3-8-27b-q4-2026-11-01T15-39-24-739Z-zzmy/metrics.json",
      "metricsSha256": "306c402cbe62cd329347b3dd917b83a8892650ceb6a2f03cb66285ce4e46b3a8",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 519,
        "sampleCount": 4,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 4,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-11-01T15:39:53.796Z",
            "sessionId": "eff5b5ac-4c91-44e6-8de3-cdd640f8ae70",
            "source": "first-token-detail",
            "tokensPerSec": 540,
            "ttftMs": 22360
          },
          {
            "line": 1620,
            "rx": "2026-11-01T15:45:01.104Z",
            "sessionId": "12c15cef-cb5a-424a-bcb6-7640b24707a6",
            "source": "first-token-detail",
            "tokensPerSec": 546,
            "ttftMs": 14479
          },
          {
            "line": 4144,
            "rx": "2026-11-01T15:53:00.682Z",
            "sessionId": "f5a34ced-bef3-49f1-a8b4-4a3269256549",
            "source": "first-token-detail",
            "tokensPerSec": 498,
            "ttftMs": 8783
          },
          {
            "line": 5786,
            "rx": "2026-11-01T15:58:37.320Z",
            "sessionId": "f5a34ced-bef3-49f1-a8b4-4a3269256549",
            "source": "first-token-detail",
            "tokensPerSec": 476,
            "ttftMs": 35394
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-craftbook-deep-security-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-deep-security-review",
        "repetition": 3
      },
      "trialId": "craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-11-01T16-27-47-991Z-iwh8",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-11-01T16-27-47-991Z-iwh8/recording/chat-events.jsonl",
      "recordingSha256": "06ac38b609c3150460106d54e18719a806a66156f03cd76126a1c747a87949e7",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-qwen3-8-27b-q4-2026-11-01T16-27-47-991Z-iwh8/metrics.json",
      "metricsSha256": "a84b30553e8e90e5cfa8346aa07e719cff43e026707a621225d10d7a415aa542",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 510,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T16:28:11.771Z",
            "sessionId": "165186f0-d247-4829-92db-d1b51a7dd182",
            "source": "first-token-detail",
            "tokensPerSec": 553,
            "ttftMs": 22135
          },
          {
            "line": 1105,
            "rx": "2026-11-01T16:31:22.338Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 9214
          },
          {
            "line": 2725,
            "rx": "2026-11-01T16:36:16.157Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 518,
            "ttftMs": 15811
          },
          {
            "line": 2841,
            "rx": "2026-11-01T16:37:01.917Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 14198
          },
          {
            "line": 2886,
            "rx": "2026-11-01T16:37:40.975Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 493,
            "ttftMs": 18483
          },
          {
            "line": 3062,
            "rx": "2026-11-01T16:38:53.470Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 510,
            "ttftMs": 22143
          },
          {
            "line": 3165,
            "rx": "2026-11-01T16:39:57.101Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 498,
            "ttftMs": 25703
          },
          {
            "line": 3199,
            "rx": "2026-11-01T16:40:03.359Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 375,
            "ttftMs": 1624
          },
          {
            "line": 3328,
            "rx": "2026-11-01T16:41:24.087Z",
            "sessionId": "76599ff8-0249-451f-ae01-c20f4a80d5b8",
            "source": "first-token-detail",
            "tokensPerSec": 524,
            "ttftMs": 31815
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-craftbook-deep-security-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-deep-security-review",
        "repetition": 3
      },
      "trialId": "craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-11-01T16-48-31-846Z-3yml",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-11-01T16-48-31-846Z-3yml/recording/chat-events.jsonl",
      "recordingSha256": "57170133ea8c77b85379916b60609900d95f583e5ade880d7fbf1d4139a23ddc",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-mlx-gemma4-12b-q4-2026-11-01T16-48-31-846Z-3yml/metrics.json",
      "metricsSha256": "aefddd36a4e25610ebb1465f8d6dcfbd562eab3288215458cedadc3609d8a3b8",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 231,
        "sampleCount": 3,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 3,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-01T16:48:47.170Z",
            "sessionId": "61f28588-0b04-4076-b51f-5ecc5401f826",
            "source": "first-token-detail",
            "tokensPerSec": 1093,
            "ttftMs": 13673
          },
          {
            "line": 1381,
            "rx": "2026-11-01T16:52:36.865Z",
            "sessionId": "61f28588-0b04-4076-b51f-5ecc5401f826",
            "source": "first-token-detail",
            "tokensPerSec": 231,
            "ttftMs": 1919
          },
          {
            "line": 1666,
            "rx": "2026-11-01T16:53:36.421Z",
            "sessionId": "b3e8a2f7-d9df-4371-8186-0f70ec918bbf",
            "source": "first-token-detail",
            "tokensPerSec": 91,
            "ttftMs": 61504
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-large-pr-review",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "large-pr-review",
        "repetition": 3
      },
      "trialId": "large-pr-review-mlx-qwen3-8-27b-q4-2026-11-01T16-55-00-774Z-vl44",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-qwen3-8-27b-q4-2026-11-01T16-55-00-774Z-vl44/recording/chat-events.jsonl",
      "recordingSha256": "69a15ed21eca71d2941f42377b0afece70a040b35f19a4cb97f40748e400b8fc",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-qwen3-8-27b-q4-2026-11-01T16-55-00-774Z-vl44/metrics.json",
      "metricsSha256": "058224d89184ff7840fa4e86bbaa81a9b3e424f48571b3b599f909a8c4da82ff",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 337,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-11-01T16:55:35.926Z",
            "sessionId": "6372fc10-2f91-4d3f-a4d3-a479759feace",
            "source": "first-token-detail",
            "tokensPerSec": 525,
            "ttftMs": 33374
          },
          {
            "line": 8185,
            "rx": "2026-11-01T17:24:49.771Z",
            "sessionId": "6372fc10-2f91-4d3f-a4d3-a479759feace",
            "source": "first-token-detail",
            "tokensPerSec": 149,
            "ttftMs": 5049
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-large-pr-review",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "large-pr-review",
        "repetition": 3
      },
      "trialId": "large-pr-review-mlx-gemma4-12b-q4-2026-11-01T18-00-41-204Z-zgpc",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-gemma4-12b-q4-2026-11-01T18-00-41-204Z-zgpc/recording/chat-events.jsonl",
      "recordingSha256": "c76ec1b53735773d72b481078a24091e5963faa9537560ae530ffe17384ebd15",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-large-pr-review/large-pr-review/large-pr-review-mlx-gemma4-12b-q4-2026-11-01T18-00-41-204Z-zgpc/metrics.json",
      "metricsSha256": "5a6001413105b0d2233b1543a9994a40262efff2c824330be535a1a8be03872c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 638,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-11-01T18:01:00.553Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 1049,
            "ttftMs": 17549
          },
          {
            "line": 2014,
            "rx": "2026-11-01T18:07:49.178Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 337,
            "ttftMs": 715
          },
          {
            "line": 3033,
            "rx": "2026-11-01T18:13:22.092Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 790,
            "ttftMs": 76053
          },
          {
            "line": 4175,
            "rx": "2026-11-01T18:18:54.378Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 735,
            "ttftMs": 94485
          },
          {
            "line": 4250,
            "rx": "2026-11-01T18:20:41.646Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 710,
            "ttftMs": 99565
          },
          {
            "line": 4996,
            "rx": "2026-11-01T18:24:35.550Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 328,
            "ttftMs": 819
          },
          {
            "line": 5098,
            "rx": "2026-11-01T18:24:51.117Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 304,
            "ttftMs": 677
          },
          {
            "line": 5242,
            "rx": "2026-11-01T18:25:28.332Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 279,
            "ttftMs": 532
          },
          {
            "line": 5422,
            "rx": "2026-11-01T18:28:13.362Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 650,
            "ttftMs": 132205
          },
          {
            "line": 6057,
            "rx": "2026-11-01T18:35:47.807Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 638,
            "ttftMs": 144253
          },
          {
            "line": 6666,
            "rx": "2026-11-01T18:41:39.985Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 440,
            "ttftMs": 13269
          },
          {
            "line": 7224,
            "rx": "2026-11-01T18:47:36.891Z",
            "sessionId": "one-shot:2600c3d0-ab18-4e31-9781-8aa121f47325",
            "source": "first-token-detail",
            "tokensPerSec": 936,
            "ttftMs": 1223
          },
          {
            "line": 7248,
            "rx": "2026-11-01T18:48:43.044Z",
            "sessionId": "one-shot:ba35b26c-eff4-4cc4-a252-aa6b36cc8d5b",
            "source": "first-token-detail",
            "tokensPerSec": 1555,
            "ttftMs": 756
          },
          {
            "line": 7446,
            "rx": "2026-11-01T18:55:01.371Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 780,
            "ttftMs": 141141
          },
          {
            "line": 9336,
            "rx": "2026-11-01T19:03:32.492Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 265,
            "ttftMs": 702
          },
          {
            "line": 9625,
            "rx": "2026-11-01T19:07:36.721Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 551,
            "ttftMs": 216865
          },
          {
            "line": 10317,
            "rx": "2026-11-01T19:14:30.400Z",
            "sessionId": "f9d8261e-5134-47ca-a29f-26450f6b8467",
            "source": "first-token-detail",
            "tokensPerSec": 554,
            "ttftMs": 227020
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-craftbook-codemod-sweep",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-codemod-sweep",
        "repetition": 3
      },
      "trialId": "craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-11-01T21-32-17-703Z-gd59",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-11-01T21-32-17-703Z-gd59/recording/chat-events.jsonl",
      "recordingSha256": "cbeb3f98b5c34c9c45aa144e9d4491a2195635fc7b4e0890693e7d90f35f141c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-gemma4-12b-q4-2026-11-01T21-32-17-703Z-gd59/metrics.json",
      "metricsSha256": "28db3f38291cc483b78e0163c2a2ead23f06b89f67354690a2f4747905173a9f",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 896,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-01T21:32:37.808Z",
            "sessionId": "9f5e0452-2b2b-4357-ae1c-233b5547b795",
            "source": "first-token-detail",
            "tokensPerSec": 1135,
            "ttftMs": 13585
          },
          {
            "line": 131,
            "rx": "2026-11-01T21:33:07.136Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 1170,
            "ttftMs": 7911
          },
          {
            "line": 285,
            "rx": "2026-11-01T21:33:46.095Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 1151,
            "ttftMs": 10061
          },
          {
            "line": 502,
            "rx": "2026-11-01T21:34:21.598Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 865,
            "ttftMs": 915
          },
          {
            "line": 4164,
            "rx": "2026-11-01T21:45:00.839Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 961,
            "ttftMs": 31281
          },
          {
            "line": 4209,
            "rx": "2026-11-01T21:45:42.558Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 936,
            "ttftMs": 33128
          },
          {
            "line": 4464,
            "rx": "2026-11-01T21:46:58.976Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 930,
            "ttftMs": 34424
          },
          {
            "line": 4578,
            "rx": "2026-11-01T21:47:24.699Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 1041
          },
          {
            "line": 5301,
            "rx": "2026-11-01T21:49:26.076Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 1144
          },
          {
            "line": 5690,
            "rx": "2026-11-01T21:51:06.473Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 924,
            "ttftMs": 38056
          },
          {
            "line": 5867,
            "rx": "2026-11-01T21:52:13.530Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 946,
            "ttftMs": 39000
          },
          {
            "line": 6292,
            "rx": "2026-11-01T21:53:28.567Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 820,
            "ttftMs": 564
          },
          {
            "line": 6621,
            "rx": "2026-11-01T21:54:25.708Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 810,
            "ttftMs": 481
          },
          {
            "line": 7072,
            "rx": "2026-11-01T21:56:28.610Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 896,
            "ttftMs": 44548
          },
          {
            "line": 7315,
            "rx": "2026-11-01T21:57:52.321Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 878,
            "ttftMs": 47605
          },
          {
            "line": 7809,
            "rx": "2026-11-01T21:59:54.522Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 863,
            "ttftMs": 48536
          },
          {
            "line": 8105,
            "rx": "2026-11-01T22:00:44.407Z",
            "sessionId": "21e8ec87-2ae0-4edd-b2d9-25d4d1ef014c",
            "source": "first-token-detail",
            "tokensPerSec": 646,
            "ttftMs": 575
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-craftbook-codemod-sweep",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-codemod-sweep",
        "repetition": 3
      },
      "trialId": "craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-11-01T22-28-39-209Z-uvix",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-11-01T22-28-39-209Z-uvix/recording/chat-events.jsonl",
      "recordingSha256": "508b0770ebd0ae6f1fed9897e4ee5e50491fcda5f4a5b6a35eaaea5f7c4e3fe0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-mlx-qwen3-8-27b-q4-2026-11-01T22-28-39-209Z-uvix/metrics.json",
      "metricsSha256": "2e79f4cace542017d5f328a4d9deae5ec50f57162fc10f503b8492ccfbec37f7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 437.5,
        "sampleCount": 24,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 24,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-01T22:29:02.236Z",
            "sessionId": "5e86c623-af03-447f-ad8e-6f941c657c54",
            "source": "first-token-detail",
            "tokensPerSec": 857,
            "ttftMs": 16477
          },
          {
            "line": 2128,
            "rx": "2026-11-01T22:35:13.034Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 17087
          },
          {
            "line": 2972,
            "rx": "2026-11-01T22:38:02.879Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 550,
            "ttftMs": 16900
          },
          {
            "line": 4821,
            "rx": "2026-11-01T22:43:27.972Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 241,
            "ttftMs": 855
          },
          {
            "line": 5148,
            "rx": "2026-11-01T22:44:45.749Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 536,
            "ttftMs": 16592
          },
          {
            "line": 6366,
            "rx": "2026-11-01T22:48:51.347Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 505,
            "ttftMs": 27924
          },
          {
            "line": 9096,
            "rx": "2026-11-01T22:57:30.736Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 493,
            "ttftMs": 27576
          },
          {
            "line": 9546,
            "rx": "2026-11-01T22:59:21.259Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 493,
            "ttftMs": 33031
          },
          {
            "line": 10451,
            "rx": "2026-11-01T23:02:43.602Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 484,
            "ttftMs": 40416
          },
          {
            "line": 11126,
            "rx": "2026-11-01T23:04:47.932Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 249,
            "ttftMs": 1386
          },
          {
            "line": 12428,
            "rx": "2026-11-01T23:08:37.729Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 235,
            "ttftMs": 1512
          },
          {
            "line": 12837,
            "rx": "2026-11-01T23:09:53.491Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 167,
            "ttftMs": 1086
          },
          {
            "line": 13552,
            "rx": "2026-11-01T23:12:49.703Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 484,
            "ttftMs": 50732
          },
          {
            "line": 15620,
            "rx": "2026-11-01T23:18:57.125Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 231,
            "ttftMs": 1694
          },
          {
            "line": 19285,
            "rx": "2026-11-01T23:30:42.926Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 440,
            "ttftMs": 74964
          },
          {
            "line": 19870,
            "rx": "2026-11-01T23:33:37.837Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 436,
            "ttftMs": 79306
          },
          {
            "line": 20655,
            "rx": "2026-11-01T23:37:13.324Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 420,
            "ttftMs": 82935
          },
          {
            "line": 22366,
            "rx": "2026-11-01T23:43:24.630Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 455,
            "ttftMs": 78196
          },
          {
            "line": 26369,
            "rx": "2026-11-01T23:55:37.017Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 180,
            "ttftMs": 1157
          },
          {
            "line": 26524,
            "rx": "2026-11-01T23:57:36.047Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 432,
            "ttftMs": 101619
          },
          {
            "line": 28039,
            "rx": "2026-11-02T00:03:43.102Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 439,
            "ttftMs": 102138
          },
          {
            "line": 28588,
            "rx": "2026-11-02T00:05:23.590Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 246,
            "ttftMs": 2240
          },
          {
            "line": 29259,
            "rx": "2026-11-02T00:07:35.451Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 150,
            "ttftMs": 1239
          },
          {
            "line": 31178,
            "rx": "2026-11-02T00:15:23.138Z",
            "sessionId": "5d92ab5b-5b52-4d6a-a6c2-469c9dd4137a",
            "source": "first-token-detail",
            "tokensPerSec": 430,
            "ttftMs": 116302
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-craftbook-refactor-module",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-refactor-module",
        "repetition": 3
      },
      "trialId": "craftbook-refactor-module-mlx-gemma4-12b-q4-2026-11-02T00-22-33-760Z-0qd0",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-gemma4-12b-q4-2026-11-02T00-22-33-760Z-0qd0/recording/chat-events.jsonl",
      "recordingSha256": "df8e47365ffec1caffc0eafce535cc07890c94f233889342a6dce5c2b3834a8e",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-gemma4-12b-q4-2026-11-02T00-22-33-760Z-0qd0/metrics.json",
      "metricsSha256": "bd07907c70a7d6a15d51e45d164f53aadf15c87d06a6284c021e3bab67a7c0e6",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1157,
        "sampleCount": 25,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 25,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-02T00:22:52.738Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1146,
            "ttftMs": 12441
          },
          {
            "line": 66,
            "rx": "2026-11-02T00:23:02.511Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 1276
          },
          {
            "line": 100,
            "rx": "2026-11-02T00:23:11.199Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 928,
            "ttftMs": 1334
          },
          {
            "line": 143,
            "rx": "2026-11-02T00:23:31.240Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1228,
            "ttftMs": 10739
          },
          {
            "line": 665,
            "rx": "2026-11-02T00:25:07.208Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1287,
            "ttftMs": 11284
          },
          {
            "line": 944,
            "rx": "2026-11-02T00:25:51.364Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 573,
            "ttftMs": 770
          },
          {
            "line": 1381,
            "rx": "2026-11-02T00:27:18.930Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1115,
            "ttftMs": 14623
          },
          {
            "line": 1419,
            "rx": "2026-11-02T00:27:29.912Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1097,
            "ttftMs": 834
          },
          {
            "line": 1437,
            "rx": "2026-11-02T00:27:34.700Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 801,
            "ttftMs": 533
          },
          {
            "line": 1475,
            "rx": "2026-11-02T00:28:15.196Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1072,
            "ttftMs": 15702
          },
          {
            "line": 1546,
            "rx": "2026-11-02T00:29:01.630Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 961,
            "ttftMs": 20557
          },
          {
            "line": 2002,
            "rx": "2026-11-02T00:30:36.393Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 992,
            "ttftMs": 20100
          },
          {
            "line": 2035,
            "rx": "2026-11-02T00:30:47.559Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1103,
            "ttftMs": 817
          },
          {
            "line": 2076,
            "rx": "2026-11-02T00:33:45.232Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1199,
            "ttftMs": 756
          },
          {
            "line": 2105,
            "rx": "2026-11-02T00:36:53.273Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1250,
            "ttftMs": 861
          },
          {
            "line": 2134,
            "rx": "2026-11-02T00:38:04.276Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1186,
            "ttftMs": 760
          },
          {
            "line": 2169,
            "rx": "2026-11-02T00:39:15.413Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1198,
            "ttftMs": 767
          },
          {
            "line": 2203,
            "rx": "2026-11-02T00:39:25.731Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1179,
            "ttftMs": 928
          },
          {
            "line": 2235,
            "rx": "2026-11-02T00:40:36.722Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1157,
            "ttftMs": 836
          },
          {
            "line": 2282,
            "rx": "2026-11-02T00:41:52.806Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1169,
            "ttftMs": 785
          },
          {
            "line": 2316,
            "rx": "2026-11-02T00:42:03.143Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1152,
            "ttftMs": 967
          },
          {
            "line": 2350,
            "rx": "2026-11-02T00:43:14.099Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1157,
            "ttftMs": 855
          },
          {
            "line": 2374,
            "rx": "2026-11-02T00:43:19.024Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1205,
            "ttftMs": 697
          },
          {
            "line": 2397,
            "rx": "2026-11-02T00:43:24.217Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1195,
            "ttftMs": 795
          },
          {
            "line": 2420,
            "rx": "2026-11-02T00:43:29.305Z",
            "sessionId": "5312b34e-5af5-4075-98c4-e2c733da7c9c",
            "source": "first-token-detail",
            "tokensPerSec": 1203,
            "ttftMs": 790
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-craftbook-refactor-module",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-refactor-module",
        "repetition": 3
      },
      "trialId": "craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-11-02T01-12-47-279Z-i210",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-11-02T01-12-47-279Z-i210/recording/chat-events.jsonl",
      "recordingSha256": "0c52b9e415cf1d5b501807abd3d9720fb78df06d372af690d8d82f5b98f7ebfb",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-mlx-qwen3-8-27b-q4-2026-11-02T01-12-47-279Z-i210/metrics.json",
      "metricsSha256": "f24e7eddd0ff31511efde2371484941938f9764db8b1c9d23037952c6736a084",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 500,
        "sampleCount": 24,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 24,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-02T01:13:10.067Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 536,
            "ttftMs": 16261
          },
          {
            "line": 159,
            "rx": "2026-11-02T01:13:31.391Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 472,
            "ttftMs": 1972
          },
          {
            "line": 295,
            "rx": "2026-11-02T01:13:53.685Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 488,
            "ttftMs": 1910
          },
          {
            "line": 469,
            "rx": "2026-11-02T01:14:40.782Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 551,
            "ttftMs": 21555
          },
          {
            "line": 2373,
            "rx": "2026-11-02T01:20:25.470Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 243,
            "ttftMs": 1131
          },
          {
            "line": 2870,
            "rx": "2026-11-02T01:21:54.011Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 280,
            "ttftMs": 1118
          },
          {
            "line": 6567,
            "rx": "2026-11-02T01:34:27.455Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 522,
            "ttftMs": 27722
          },
          {
            "line": 6883,
            "rx": "2026-11-02T01:35:22.686Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 426,
            "ttftMs": 2191
          },
          {
            "line": 7253,
            "rx": "2026-11-02T01:37:01.759Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 524,
            "ttftMs": 33206
          },
          {
            "line": 7326,
            "rx": "2026-11-02T01:37:14.312Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 419,
            "ttftMs": 756
          },
          {
            "line": 7394,
            "rx": "2026-11-02T01:38:24.645Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 494,
            "ttftMs": 30570
          },
          {
            "line": 7511,
            "rx": "2026-11-02T01:39:46.616Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 510,
            "ttftMs": 31585
          },
          {
            "line": 7669,
            "rx": "2026-11-02T01:40:17.056Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 464,
            "ttftMs": 1315
          },
          {
            "line": 7737,
            "rx": "2026-11-02T01:43:37.313Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 745,
            "ttftMs": 24002
          },
          {
            "line": 7879,
            "rx": "2026-11-02T01:46:57.858Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 560,
            "ttftMs": 1262
          },
          {
            "line": 7973,
            "rx": "2026-11-02T01:50:10.818Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 560,
            "ttftMs": 1264
          },
          {
            "line": 8074,
            "rx": "2026-11-02T01:51:31.778Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 498,
            "ttftMs": 992
          },
          {
            "line": 8169,
            "rx": "2026-11-02T01:53:41.335Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 707,
            "ttftMs": 29086
          },
          {
            "line": 8195,
            "rx": "2026-11-02T01:54:50.408Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 459,
            "ttftMs": 930
          },
          {
            "line": 8345,
            "rx": "2026-11-02T01:56:17.946Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 502,
            "ttftMs": 1080
          },
          {
            "line": 8459,
            "rx": "2026-11-02T01:57:40.209Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 496,
            "ttftMs": 1083
          },
          {
            "line": 8563,
            "rx": "2026-11-02T01:59:02.356Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 464,
            "ttftMs": 1003
          },
          {
            "line": 8644,
            "rx": "2026-11-02T02:00:21.162Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 604,
            "ttftMs": 38730
          },
          {
            "line": 8726,
            "rx": "2026-11-02T02:01:56.992Z",
            "sessionId": "5f564a1e-4276-4fb9-bca0-9ec617f977f5",
            "source": "first-token-detail",
            "tokensPerSec": 526,
            "ttftMs": 47131
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-developer-craftbook-bug-fix-tdd",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "developer",
        "scenario": "craftbook-bug-fix-tdd",
        "repetition": 3
      },
      "trialId": "craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-11-02T03-16-42-447Z-3e1d",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-11-02T03-16-42-447Z-3e1d/recording/chat-events.jsonl",
      "recordingSha256": "419fe923343a0d1f3c47ae4240cfebe510378eda339a7c814aa6e64d01f68cb5",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-qwen3-8-27b-q4-2026-11-02T03-16-42-447Z-3e1d/metrics.json",
      "metricsSha256": "9976ae949802181d5e68083e2a6f9a29252bb7f3a4b0663e56d3664ad98dead4",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 514,
        "sampleCount": 24,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 24,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 33,
            "rx": "2026-11-02T03:17:11.968Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 549,
            "ttftMs": 23004
          },
          {
            "line": 456,
            "rx": "2026-11-02T03:18:21.205Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 276,
            "ttftMs": 754
          },
          {
            "line": 1101,
            "rx": "2026-11-02T03:20:30.416Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 542,
            "ttftMs": 20550
          },
          {
            "line": 1409,
            "rx": "2026-11-02T03:21:43.609Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 18927
          },
          {
            "line": 1518,
            "rx": "2026-11-02T03:22:14.796Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 475,
            "ttftMs": 14817
          },
          {
            "line": 2541,
            "rx": "2026-11-02T03:25:23.145Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 311,
            "ttftMs": 1677
          },
          {
            "line": 3566,
            "rx": "2026-11-02T03:28:29.658Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 272,
            "ttftMs": 1189
          },
          {
            "line": 3660,
            "rx": "2026-11-02T03:29:05.178Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 499,
            "ttftMs": 23799
          },
          {
            "line": 3997,
            "rx": "2026-11-02T03:30:30.259Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 26181
          },
          {
            "line": 4100,
            "rx": "2026-11-02T03:30:50.533Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 482,
            "ttftMs": 1134
          },
          {
            "line": 4231,
            "rx": "2026-11-02T03:31:15.764Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 358,
            "ttftMs": 874
          },
          {
            "line": 4289,
            "rx": "2026-11-02T03:32:25.449Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 29701
          },
          {
            "line": 4368,
            "rx": "2026-11-02T03:35:20.405Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 756,
            "ttftMs": 21868
          },
          {
            "line": 4483,
            "rx": "2026-11-02T03:35:41.631Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 472,
            "ttftMs": 1304
          },
          {
            "line": 4558,
            "rx": "2026-11-02T03:39:08.098Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 741,
            "ttftMs": 24809
          },
          {
            "line": 4693,
            "rx": "2026-11-02T03:40:34.103Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 514,
            "ttftMs": 1048
          },
          {
            "line": 4739,
            "rx": "2026-11-02T03:42:32.712Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 726,
            "ttftMs": 27181
          },
          {
            "line": 4856,
            "rx": "2026-11-02T03:43:53.962Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 514,
            "ttftMs": 1230
          },
          {
            "line": 4967,
            "rx": "2026-11-02T03:45:15.474Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 484,
            "ttftMs": 1086
          },
          {
            "line": 5088,
            "rx": "2026-11-02T03:46:37.138Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 484,
            "ttftMs": 1068
          },
          {
            "line": 5217,
            "rx": "2026-11-02T03:48:31.957Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 722,
            "ttftMs": 34169
          },
          {
            "line": 5268,
            "rx": "2026-11-02T03:50:48.863Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 709,
            "ttftMs": 31887
          },
          {
            "line": 5344,
            "rx": "2026-11-02T03:52:15.297Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 597,
            "ttftMs": 39908
          },
          {
            "line": 5540,
            "rx": "2026-11-02T03:54:31.649Z",
            "sessionId": "46d7b27b-c12c-46ac-b11c-3fbfa664c639",
            "source": "first-token-detail",
            "tokensPerSec": 409,
            "ttftMs": 61156
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-developer-craftbook-bug-fix-tdd",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "developer",
        "scenario": "craftbook-bug-fix-tdd",
        "repetition": 3
      },
      "trialId": "craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-11-02T04-50-37-295Z-n74w",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-11-02T04-50-37-295Z-n74w/recording/chat-events.jsonl",
      "recordingSha256": "527d98a02cc807d1e77b8d18d11d13d07a2a348d996f81a0acf789a9c8a08039",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-mlx-gemma4-12b-q4-2026-11-02T04-50-37-295Z-n74w/metrics.json",
      "metricsSha256": "7f6c66c1187d3c458d8fe57105995d07ea4161ecb78c19e66479ee4efe96c810",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1017,
        "sampleCount": 28,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 28,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-02T04:50:57.142Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1191,
            "ttftMs": 13214
          },
          {
            "line": 184,
            "rx": "2026-11-02T04:51:32.736Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1215,
            "ttftMs": 8056
          },
          {
            "line": 213,
            "rx": "2026-11-02T04:51:39.215Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 946,
            "ttftMs": 1102
          },
          {
            "line": 242,
            "rx": "2026-11-02T04:51:45.697Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1038,
            "ttftMs": 766
          },
          {
            "line": 268,
            "rx": "2026-11-02T04:51:59.160Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1219,
            "ttftMs": 9951
          },
          {
            "line": 448,
            "rx": "2026-11-02T04:52:28.889Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 688,
            "ttftMs": 708
          },
          {
            "line": 862,
            "rx": "2026-11-02T04:53:48.018Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1189,
            "ttftMs": 13371
          },
          {
            "line": 2943,
            "rx": "2026-11-02T04:59:51.618Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 23157
          },
          {
            "line": 2976,
            "rx": "2026-11-02T05:00:01.207Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1023,
            "ttftMs": 843
          },
          {
            "line": 3009,
            "rx": "2026-11-02T05:00:32.173Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1019,
            "ttftMs": 26247
          },
          {
            "line": 3542,
            "rx": "2026-11-02T05:02:25.479Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 27067
          },
          {
            "line": 3573,
            "rx": "2026-11-02T05:02:34.402Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1002,
            "ttftMs": 846
          },
          {
            "line": 3612,
            "rx": "2026-11-02T05:03:09.385Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 962,
            "ttftMs": 30280
          },
          {
            "line": 4003,
            "rx": "2026-11-02T05:04:42.789Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 956,
            "ttftMs": 31718
          },
          {
            "line": 4043,
            "rx": "2026-11-02T05:04:57.294Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1050,
            "ttftMs": 903
          },
          {
            "line": 4083,
            "rx": "2026-11-02T05:05:36.507Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 941,
            "ttftMs": 34517
          },
          {
            "line": 4524,
            "rx": "2026-11-02T05:07:24.396Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 952,
            "ttftMs": 35211
          },
          {
            "line": 4578,
            "rx": "2026-11-02T05:08:11.331Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 955,
            "ttftMs": 36378
          },
          {
            "line": 4620,
            "rx": "2026-11-02T05:09:12.052Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 977,
            "ttftMs": 35778
          },
          {
            "line": 4655,
            "rx": "2026-11-02T05:10:23.845Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1113,
            "ttftMs": 946
          },
          {
            "line": 4684,
            "rx": "2026-11-02T05:11:35.231Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1053,
            "ttftMs": 816
          },
          {
            "line": 4722,
            "rx": "2026-11-02T05:12:46.678Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1015,
            "ttftMs": 755
          },
          {
            "line": 4768,
            "rx": "2026-11-02T05:13:26.874Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1318,
            "ttftMs": 30201
          },
          {
            "line": 5033,
            "rx": "2026-11-02T05:14:44.704Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1144,
            "ttftMs": 35771
          },
          {
            "line": 5054,
            "rx": "2026-11-02T05:14:51.359Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 733,
            "ttftMs": 1015
          },
          {
            "line": 5097,
            "rx": "2026-11-02T05:15:35.822Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 1066,
            "ttftMs": 40230
          },
          {
            "line": 5432,
            "rx": "2026-11-02T05:17:17.365Z",
            "sessionId": "043aebe1-46b3-4b64-95fc-00a516102040",
            "source": "first-token-detail",
            "tokensPerSec": 831,
            "ttftMs": 52591
          },
          {
            "line": 5455,
            "rx": "2026-11-02T05:20:39.381Z",
            "sessionId": "one-shot:2da9c920-33e8-4880-ab47-f1908c297619",
            "source": "first-token-detail",
            "tokensPerSec": 1684,
            "ttftMs": 891
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-find-vs-create",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-find-vs-create",
        "repetition": 3
      },
      "trialId": "craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-11-02T05-22-30-018Z-aq07",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-11-02T05-22-30-018Z-aq07/recording/chat-events.jsonl",
      "recordingSha256": "da70daa89dbc519c6a3a1fc4581ec73a479e732a199e3ed7f40c0928d4dc9c0c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-qwen3-8-27b-q4-2026-11-02T05-22-30-018Z-aq07/metrics.json",
      "metricsSha256": "911f6d610ffea1bbbb2b64e2a116080eacccc8b5a8c42df95193b5642963ce1a",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 807,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 28,
            "rx": "2026-11-02T05:22:51.501Z",
            "sessionId": "36d0d287-0244-4b8e-9131-d947f1d3d61b",
            "source": "first-token-detail",
            "tokensPerSec": 807,
            "ttftMs": 19788
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-find-vs-create",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-find-vs-create",
        "repetition": 3
      },
      "trialId": "craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-11-02T05-48-51-395Z-5dc5",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-11-02T05-48-51-395Z-5dc5/recording/chat-events.jsonl",
      "recordingSha256": "1c89fa9f8d82287ded5d71ac730ae6f451991338720eee1530639c468f1f933b",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-mlx-gemma4-12b-q4-2026-11-02T05-48-51-395Z-5dc5/metrics.json",
      "metricsSha256": "6cbae6082ce72134b53809fc1a3b66aee97cff56e5bc486b658a115f336bd594",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 694.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 27,
            "rx": "2026-11-02T05:49:08.511Z",
            "sessionId": "85366a08-86e7-432b-98b7-6819908ebaef",
            "source": "first-token-detail",
            "tokensPerSec": 1117,
            "ttftMs": 15376
          },
          {
            "line": 1552,
            "rx": "2026-11-02T05:53:28.180Z",
            "sessionId": "85366a08-86e7-432b-98b7-6819908ebaef",
            "source": "first-token-detail",
            "tokensPerSec": 272,
            "ttftMs": 409
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-edit-midtask",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-edit-midtask",
        "repetition": 3
      },
      "trialId": "craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-11-02T06-05-17-170Z-axk4",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-11-02T06-05-17-170Z-axk4/recording/chat-events.jsonl",
      "recordingSha256": "d5ff8c70ed52621f3239e8b0b5e4691fc1c41b152be3a01e985c3dfe91b9c267",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-gemma4-12b-q4-2026-11-02T06-05-17-170Z-axk4/metrics.json",
      "metricsSha256": "e39368f22081e37a94cb3629839db5256cb359cd0f638e89d150f51263e75094",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 547.5,
        "sampleCount": 18,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 18,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-11-02T06:05:37.762Z",
            "sessionId": "e7118ce0-20c6-42da-80d4-be4b858a1c5a",
            "source": "first-token-detail",
            "tokensPerSec": 1074,
            "ttftMs": 18910
          },
          {
            "line": 317,
            "rx": "2026-11-02T06:06:42.577Z",
            "sessionId": "e7118ce0-20c6-42da-80d4-be4b858a1c5a",
            "source": "first-token-detail",
            "tokensPerSec": 525,
            "ttftMs": 672
          },
          {
            "line": 392,
            "rx": "2026-11-02T06:06:53.007Z",
            "sessionId": "e7118ce0-20c6-42da-80d4-be4b858a1c5a",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 803
          },
          {
            "line": 760,
            "rx": "2026-11-02T06:08:16.725Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 1050,
            "ttftMs": 11549
          },
          {
            "line": 849,
            "rx": "2026-11-02T06:08:32.614Z",
            "sessionId": "e7118ce0-20c6-42da-80d4-be4b858a1c5a",
            "source": "first-token-detail",
            "tokensPerSec": 429,
            "ttftMs": 412
          },
          {
            "line": 1083,
            "rx": "2026-11-02T06:09:07.172Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 466,
            "ttftMs": 824
          },
          {
            "line": 1408,
            "rx": "2026-11-02T06:10:03.238Z",
            "sessionId": "e7118ce0-20c6-42da-80d4-be4b858a1c5a",
            "source": "first-token-detail",
            "tokensPerSec": 476,
            "ttftMs": 489
          },
          {
            "line": 1661,
            "rx": "2026-11-02T06:10:45.192Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 371,
            "ttftMs": 574
          },
          {
            "line": 1776,
            "rx": "2026-11-02T06:11:01.834Z",
            "sessionId": "e7118ce0-20c6-42da-80d4-be4b858a1c5a",
            "source": "first-token-detail",
            "tokensPerSec": 424,
            "ttftMs": 526
          },
          {
            "line": 2050,
            "rx": "2026-11-02T06:11:43.889Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 335,
            "ttftMs": 473
          },
          {
            "line": 3679,
            "rx": "2026-11-02T06:16:17.793Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 872,
            "ttftMs": 9567
          },
          {
            "line": 3883,
            "rx": "2026-11-02T06:16:51.440Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 473,
            "ttftMs": 825
          },
          {
            "line": 4044,
            "rx": "2026-11-02T06:17:19.156Z",
            "sessionId": "a161fddf-fddf-45c7-8b33-be464e285d7f",
            "source": "first-token-detail",
            "tokensPerSec": 1070,
            "ttftMs": 3472
          },
          {
            "line": 4382,
            "rx": "2026-11-02T06:18:15.378Z",
            "sessionId": "20e9c20a-2916-4af6-afa5-c385451a3adb",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 541
          },
          {
            "line": 4487,
            "rx": "2026-11-02T06:18:36.250Z",
            "sessionId": "a161fddf-fddf-45c7-8b33-be464e285d7f",
            "source": "first-token-detail",
            "tokensPerSec": 1053,
            "ttftMs": 6689
          },
          {
            "line": 4570,
            "rx": "2026-11-02T06:19:08.250Z",
            "sessionId": "a161fddf-fddf-45c7-8b33-be464e285d7f",
            "source": "first-token-detail",
            "tokensPerSec": 1048,
            "ttftMs": 12364
          },
          {
            "line": 4803,
            "rx": "2026-11-02T06:19:48.853Z",
            "sessionId": "4350cd39-99d5-413c-9b06-8e3fe0b033f5",
            "source": "first-token-detail",
            "tokensPerSec": 1057,
            "ttftMs": 14664
          },
          {
            "line": 4896,
            "rx": "2026-11-02T06:20:07.399Z",
            "sessionId": "a161fddf-fddf-45c7-8b33-be464e285d7f",
            "source": "first-token-detail",
            "tokensPerSec": 1022,
            "ttftMs": 20903
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-edit-midtask",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-edit-midtask",
        "repetition": 3
      },
      "trialId": "craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-11-02T06-31-52-526Z-b0il",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-11-02T06-31-52-526Z-b0il/recording/chat-events.jsonl",
      "recordingSha256": "03149490645dcfba3b2acba78314965886302365849a0db2074d9981c945f230",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-mlx-qwen3-8-27b-q4-2026-11-02T06-31-52-526Z-b0il/metrics.json",
      "metricsSha256": "d6ef56c53c5e5ea4f5947bf31e637fcabe6897c48df147460486cc4785f515fb",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 291,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 40,
            "rx": "2026-11-02T06:32:35.609Z",
            "sessionId": "08039a83-9605-447b-83aa-bc826107e1bb",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 41381
          },
          {
            "line": 3234,
            "rx": "2026-11-02T06:42:15.039Z",
            "sessionId": "08039a83-9605-447b-83aa-bc826107e1bb",
            "source": "first-token-detail",
            "tokensPerSec": 283,
            "ttftMs": 1875
          },
          {
            "line": 4548,
            "rx": "2026-11-02T06:46:16.565Z",
            "sessionId": "bf554ad5-f5ff-4445-946d-4f10f0f31aa0",
            "source": "first-token-detail",
            "tokensPerSec": 515,
            "ttftMs": 7870
          },
          {
            "line": 4645,
            "rx": "2026-11-02T06:46:31.247Z",
            "sessionId": "08039a83-9605-447b-83aa-bc826107e1bb",
            "source": "first-token-detail",
            "tokensPerSec": 291,
            "ttftMs": 1609
          },
          {
            "line": 4840,
            "rx": "2026-11-02T06:47:12.046Z",
            "sessionId": "bf554ad5-f5ff-4445-946d-4f10f0f31aa0",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 7567
          },
          {
            "line": 4885,
            "rx": "2026-11-02T06:47:19.039Z",
            "sessionId": "08039a83-9605-447b-83aa-bc826107e1bb",
            "source": "first-token-detail",
            "tokensPerSec": 248,
            "ttftMs": 1222
          },
          {
            "line": 4948,
            "rx": "2026-11-02T06:47:28.570Z",
            "sessionId": "08039a83-9605-447b-83aa-bc826107e1bb",
            "source": "first-token-detail",
            "tokensPerSec": 234,
            "ttftMs": 1378
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-invoice-run",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-invoice-run",
        "repetition": 3
      },
      "trialId": "craftbook-invoice-run-mlx-gemma4-12b-q4-2026-11-02T06-48-06-622Z-syhp",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-gemma4-12b-q4-2026-11-02T06-48-06-622Z-syhp/recording/chat-events.jsonl",
      "recordingSha256": "37071ba22af570ea5d017667d60141b6c4f1f11d6f02b124ea0b40771ea77975",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-gemma4-12b-q4-2026-11-02T06-48-06-622Z-syhp/metrics.json",
      "metricsSha256": "b45217f6484d387536147a4edd76351cec5766be2786cc2fda348e2924b9437c",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 1009,
        "sampleCount": 23,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 23,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-02T06:48:26.759Z",
            "sessionId": "f166c2ef-6db3-4157-97f7-4bf84baa9536",
            "source": "first-token-detail",
            "tokensPerSec": 1160,
            "ttftMs": 13626
          },
          {
            "line": 143,
            "rx": "2026-11-02T06:48:58.800Z",
            "sessionId": "b2441379-5cf4-4914-86bf-b0db83946c82",
            "source": "first-token-detail",
            "tokensPerSec": 1486,
            "ttftMs": 5652
          },
          {
            "line": 315,
            "rx": "2026-11-02T06:49:49.033Z",
            "sessionId": "2d72bc92-65b3-4b3b-8454-c4bb17205780",
            "source": "first-token-detail",
            "tokensPerSec": 1444,
            "ttftMs": 5836
          },
          {
            "line": 494,
            "rx": "2026-11-02T06:50:38.874Z",
            "sessionId": "ba99f5f2-71eb-4522-bba8-2e11a75d2dfb",
            "source": "first-token-detail",
            "tokensPerSec": 1497,
            "ttftMs": 5631
          },
          {
            "line": 721,
            "rx": "2026-11-02T06:51:40.852Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 1161,
            "ttftMs": 7583
          },
          {
            "line": 882,
            "rx": "2026-11-02T06:52:29.895Z",
            "sessionId": "ad79b0ee-5994-4ab9-873b-8c02c03619a6",
            "source": "first-token-detail",
            "tokensPerSec": 1264,
            "ttftMs": 6703
          },
          {
            "line": 1114,
            "rx": "2026-11-02T06:53:35.356Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 1061,
            "ttftMs": 15833
          },
          {
            "line": 1191,
            "rx": "2026-11-02T06:53:57.682Z",
            "sessionId": "3fe0dd46-1382-4d73-acde-b0230d2318f8",
            "source": "first-token-detail",
            "tokensPerSec": 219,
            "ttftMs": 39396
          },
          {
            "line": 1216,
            "rx": "2026-11-02T06:54:00.487Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 1247
          },
          {
            "line": 1357,
            "rx": "2026-11-02T06:55:12.689Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 986,
            "ttftMs": 18185
          },
          {
            "line": 1465,
            "rx": "2026-11-02T06:55:50.909Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 982,
            "ttftMs": 20574
          },
          {
            "line": 1807,
            "rx": "2026-11-02T06:56:43.903Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 463,
            "ttftMs": 516
          },
          {
            "line": 1890,
            "rx": "2026-11-02T06:56:53.413Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 408,
            "ttftMs": 565
          },
          {
            "line": 2185,
            "rx": "2026-11-02T06:57:37.306Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 438,
            "ttftMs": 667
          },
          {
            "line": 2589,
            "rx": "2026-11-02T06:58:55.939Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 1003,
            "ttftMs": 20321
          },
          {
            "line": 2698,
            "rx": "2026-11-02T06:59:30.634Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 1003,
            "ttftMs": 21486
          },
          {
            "line": 2885,
            "rx": "2026-11-02T07:00:24.705Z",
            "sessionId": "e0604e91-2e7b-4cb3-892f-30c1188d345e",
            "source": "first-token-detail",
            "tokensPerSec": 1190,
            "ttftMs": 6262
          },
          {
            "line": 2966,
            "rx": "2026-11-02T07:00:37.342Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 655,
            "ttftMs": 1151
          },
          {
            "line": 3574,
            "rx": "2026-11-02T07:02:35.073Z",
            "sessionId": "51bd80bf-149f-451d-bec1-86d166cf25fc",
            "source": "first-token-detail",
            "tokensPerSec": 1123,
            "ttftMs": 6616
          },
          {
            "line": 3684,
            "rx": "2026-11-02T07:03:24.499Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 1009,
            "ttftMs": 22559
          },
          {
            "line": 3793,
            "rx": "2026-11-02T07:04:26.487Z",
            "sessionId": "c224c7d3-1c60-4cdf-83e4-39656712a5f4",
            "source": "first-token-detail",
            "tokensPerSec": 985,
            "ttftMs": 24275
          },
          {
            "line": 3888,
            "rx": "2026-11-02T07:05:16.206Z",
            "sessionId": "3515197e-6c0e-42df-b56f-cd8622e1e9d8",
            "source": "first-token-detail",
            "tokensPerSec": 1158,
            "ttftMs": 7798
          },
          {
            "line": 4114,
            "rx": "2026-11-02T07:07:12.380Z",
            "sessionId": "025c619b-ae7d-4f4e-9516-709914be8348",
            "source": "first-token-detail",
            "tokensPerSec": 1898,
            "ttftMs": 3810
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-invoice-run",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-invoice-run",
        "repetition": 3
      },
      "trialId": "craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-11-02T07-12-29-556Z-ei5a",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-11-02T07-12-29-556Z-ei5a/recording/chat-events.jsonl",
      "recordingSha256": "36566e5b8c63b16c17609427a7567f80f690936fa403a684637b6f8348cd08f7",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-mlx-qwen3-8-27b-q4-2026-11-02T07-12-29-556Z-ei5a/metrics.json",
      "metricsSha256": "5cd2fe637c2adeac5209f2f108e47993025baa4a690bf941ccb6f5abf0058226",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 555.5,
        "sampleCount": 6,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 6,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-11-02T07:12:56.281Z",
            "sessionId": "06361359-b168-41e7-bb16-0e5c4e814480",
            "source": "first-token-detail",
            "tokensPerSec": 607,
            "ttftMs": 20219
          },
          {
            "line": 387,
            "rx": "2026-11-02T07:14:03.408Z",
            "sessionId": "e7487160-f905-412b-ad3e-de62693fa00b",
            "source": "first-token-detail",
            "tokensPerSec": 631,
            "ttftMs": 12239
          },
          {
            "line": 835,
            "rx": "2026-11-02T07:15:30.173Z",
            "sessionId": "2b989fd9-a8f6-426c-ac3d-3d28491c2018",
            "source": "first-token-detail",
            "tokensPerSec": 553,
            "ttftMs": 13993
          },
          {
            "line": 1322,
            "rx": "2026-11-02T07:17:15.064Z",
            "sessionId": "d9e7e9f4-502e-4c68-b7d4-fb242ef11f03",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 13870
          },
          {
            "line": 2364,
            "rx": "2026-11-02T07:20:46.864Z",
            "sessionId": "677f2de9-3d4f-4d70-b0e1-dcf9af827b59",
            "source": "first-token-detail",
            "tokensPerSec": 538,
            "ttftMs": 15610
          },
          {
            "line": 2859,
            "rx": "2026-11-02T07:22:52.818Z",
            "sessionId": "6768cfcf-d753-4071-b408-991da69539d9",
            "source": "first-token-detail",
            "tokensPerSec": 519,
            "ttftMs": 16546
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-author-linear",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-linear",
        "repetition": 3
      },
      "trialId": "craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-11-02T08-00-21-017Z-wtzc",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-11-02T08-00-21-017Z-wtzc/recording/chat-events.jsonl",
      "recordingSha256": "0c1a337a56a1aec89dc25c0a7fd5c6a75d20d51663ea411708e589e5749a72e8",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-qwen3-8-27b-q4-2026-11-02T08-00-21-017Z-wtzc/metrics.json",
      "metricsSha256": "76db38af62124571b596b789366a0de01ea875511cdaba917d957d7d5101f278",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 503,
        "sampleCount": 17,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 17,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 37,
            "rx": "2026-11-02T08:01:02.391Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 546,
            "ttftMs": 39668
          },
          {
            "line": 6608,
            "rx": "2026-11-02T08:21:04.436Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 288,
            "ttftMs": 9954
          },
          {
            "line": 7046,
            "rx": "2026-11-02T08:22:32.163Z",
            "sessionId": "fdb67490-12ee-4a22-8e9b-00b7a89ce29e",
            "source": "first-token-detail",
            "tokensPerSec": 92,
            "ttftMs": 134051
          },
          {
            "line": 7759,
            "rx": "2026-11-02T08:25:09.691Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 268,
            "ttftMs": 3575
          },
          {
            "line": 13514,
            "rx": "2026-11-02T08:43:07.060Z",
            "sessionId": "e23b299b-2d79-4588-a2d0-cc714de7be80",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 29400
          },
          {
            "line": 13902,
            "rx": "2026-11-02T08:44:23.355Z",
            "sessionId": "84351f37-9b5d-4574-8b64-6e467dddfd2d",
            "source": "first-token-detail",
            "tokensPerSec": 491,
            "ttftMs": 44685
          },
          {
            "line": 15397,
            "rx": "2026-11-02T08:48:51.279Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 222,
            "ttftMs": 8236
          },
          {
            "line": 16933,
            "rx": "2026-11-02T08:54:17.266Z",
            "sessionId": "b9a92aa2-9b11-45c7-8b35-858363d92103",
            "source": "first-token-detail",
            "tokensPerSec": 520,
            "ttftMs": 18190
          },
          {
            "line": 17184,
            "rx": "2026-11-02T08:55:01.640Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 202,
            "ttftMs": 17854
          },
          {
            "line": 17602,
            "rx": "2026-11-02T08:56:41.995Z",
            "sessionId": "bcb252bb-d22c-48bf-bb4f-b11c874a2fb9",
            "source": "first-token-detail",
            "tokensPerSec": 508,
            "ttftMs": 98074
          },
          {
            "line": 20189,
            "rx": "2026-11-02T09:06:19.809Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 185,
            "ttftMs": 7461
          },
          {
            "line": 22230,
            "rx": "2026-11-02T09:13:48.466Z",
            "sessionId": "9f3aa8ba-7cc2-4813-b889-57ba75cdc048",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 20648
          },
          {
            "line": 22573,
            "rx": "2026-11-02T09:14:45.970Z",
            "sessionId": "9f3aa8ba-7cc2-4813-b889-57ba75cdc048",
            "source": "first-token-detail",
            "tokensPerSec": 538,
            "ttftMs": 13214
          },
          {
            "line": 22946,
            "rx": "2026-11-02T09:15:54.903Z",
            "sessionId": "9f3aa8ba-7cc2-4813-b889-57ba75cdc048",
            "source": "first-token-detail",
            "tokensPerSec": 531,
            "ttftMs": 15116
          },
          {
            "line": 23229,
            "rx": "2026-11-02T09:16:51.487Z",
            "sessionId": "b9a92aa2-9b11-45c7-8b35-858363d92103",
            "source": "first-token-detail",
            "tokensPerSec": 527,
            "ttftMs": 139768
          },
          {
            "line": 23420,
            "rx": "2026-11-02T09:17:37.893Z",
            "sessionId": "ebb54081-419b-407e-bffc-6b1e4007bf8b",
            "source": "first-token-detail",
            "tokensPerSec": 188,
            "ttftMs": 3141
          },
          {
            "line": 23648,
            "rx": "2026-11-02T09:18:47.740Z",
            "sessionId": "bcb252bb-d22c-48bf-bb4f-b11c874a2fb9",
            "source": "first-token-detail",
            "tokensPerSec": 503,
            "ttftMs": 63265
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-author-linear",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-linear",
        "repetition": 3
      },
      "trialId": "craftbook-author-linear-mlx-gemma4-12b-q4-2026-11-02T10-13-18-192Z-yodh",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-gemma4-12b-q4-2026-11-02T10-13-18-192Z-yodh/recording/chat-events.jsonl",
      "recordingSha256": "02b8009440a95fc2ab05025005d0347ba60178335315dab81f5e28e21332b8a8",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-mlx-gemma4-12b-q4-2026-11-02T10-13-18-192Z-yodh/metrics.json",
      "metricsSha256": "554329ec53cae6d4cc480b91c6c886c0e32ae349275446a59b7ece475b8792c3",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 903,
        "sampleCount": 7,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 7,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-02T10:13:38.446Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 1132,
            "ttftMs": 18576
          },
          {
            "line": 198,
            "rx": "2026-11-02T10:14:15.961Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 656,
            "ttftMs": 657
          },
          {
            "line": 518,
            "rx": "2026-11-02T10:15:10.801Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 903,
            "ttftMs": 556
          },
          {
            "line": 705,
            "rx": "2026-11-02T10:23:51.417Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 813,
            "ttftMs": 6568
          },
          {
            "line": 1175,
            "rx": "2026-11-02T10:25:36.595Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 1000,
            "ttftMs": 950
          },
          {
            "line": 1667,
            "rx": "2026-11-02T10:26:51.775Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 845,
            "ttftMs": 500
          },
          {
            "line": 1860,
            "rx": "2026-11-02T10:27:27.391Z",
            "sessionId": "a88a3750-5976-45f0-b66d-0d382eb221b4",
            "source": "first-token-detail",
            "tokensPerSec": 945,
            "ttftMs": 820
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-route-multi",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-route-multi",
        "repetition": 3
      },
      "trialId": "craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-11-02T10-33-04-013Z-5lji",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-11-02T10-33-04-013Z-5lji/recording/chat-events.jsonl",
      "recordingSha256": "f9f5603a33fb1a65dc27b2c86e9a89bf6877c40d90a91c27ccbba849574f6d7d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-qwen3-8-27b-q4-2026-11-02T10-33-04-013Z-5lji/metrics.json",
      "metricsSha256": "c018af9492a30bb9f6d8f7b589d9394019ece8cca75da06d49a77685578b47dd",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 554,
        "sampleCount": 9,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 9,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-02T10:33:33.049Z",
            "sessionId": "eb0feca0-3591-4a24-8b25-b73baaeaf32a",
            "source": "first-token-detail",
            "tokensPerSec": 572,
            "ttftMs": 27306
          },
          {
            "line": 1901,
            "rx": "2026-11-02T10:38:53.017Z",
            "sessionId": "949a5a51-04e8-4584-b0e1-f789bd6130c8",
            "source": "first-token-detail",
            "tokensPerSec": 512,
            "ttftMs": 112268
          },
          {
            "line": 2144,
            "rx": "2026-11-02T10:39:36.847Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 58334
          },
          {
            "line": 3108,
            "rx": "2026-11-02T10:42:24.154Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 685,
            "ttftMs": 5520
          },
          {
            "line": 3184,
            "rx": "2026-11-02T10:42:42.813Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 718,
            "ttftMs": 4026
          },
          {
            "line": 3257,
            "rx": "2026-11-02T10:43:06.623Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 554,
            "ttftMs": 7675
          },
          {
            "line": 3347,
            "rx": "2026-11-02T10:43:37.833Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 589,
            "ttftMs": 8673
          },
          {
            "line": 3435,
            "rx": "2026-11-02T10:44:11.608Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 522,
            "ttftMs": 12247
          },
          {
            "line": 3635,
            "rx": "2026-11-02T10:45:08.784Z",
            "sessionId": "4d7b2da6-1d2b-4353-8766-1a56cc5ce80d",
            "source": "first-token-detail",
            "tokensPerSec": 550,
            "ttftMs": 14067
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-route-multi",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-route-multi",
        "repetition": 3
      },
      "trialId": "craftbook-route-multi-mlx-gemma4-12b-q4-2026-11-02T11-10-42-097Z-fgan",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-gemma4-12b-q4-2026-11-02T11-10-42-097Z-fgan/recording/chat-events.jsonl",
      "recordingSha256": "f5d52e6f9ca959f38fcd8589e179472c05a7706dfb1aca907c5de1047f806689",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-mlx-gemma4-12b-q4-2026-11-02T11-10-42-097Z-fgan/metrics.json",
      "metricsSha256": "012756b63ec632793794594238dbc1204d358fdb8c57e12874423e54ef918bea",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 712,
        "sampleCount": 22,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 22,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 26,
            "rx": "2026-11-02T11:10:56.822Z",
            "sessionId": "6d07780c-f5d3-4608-b147-d87f0eb34f40",
            "source": "first-token-detail",
            "tokensPerSec": 1510,
            "ttftMs": 12998
          },
          {
            "line": 1470,
            "rx": "2026-11-02T11:14:49.424Z",
            "sessionId": "6d07780c-f5d3-4608-b147-d87f0eb34f40",
            "source": "first-token-detail",
            "tokensPerSec": 240,
            "ttftMs": 460
          },
          {
            "line": 1676,
            "rx": "2026-11-02T11:15:21.842Z",
            "sessionId": "6d07780c-f5d3-4608-b147-d87f0eb34f40",
            "source": "first-token-detail",
            "tokensPerSec": 363,
            "ttftMs": 520
          },
          {
            "line": 2195,
            "rx": "2026-11-02T11:16:46.003Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 671,
            "ttftMs": 21977
          },
          {
            "line": 2394,
            "rx": "2026-11-02T11:17:20.520Z",
            "sessionId": "f1fbb76e-d4fd-459b-8517-56c4c163a40d",
            "source": "first-token-detail",
            "tokensPerSec": 1011,
            "ttftMs": 41370
          },
          {
            "line": 3902,
            "rx": "2026-11-02T11:21:20.825Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 415,
            "ttftMs": 1319
          },
          {
            "line": 4062,
            "rx": "2026-11-02T11:21:50.711Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 558,
            "ttftMs": 13257
          },
          {
            "line": 4221,
            "rx": "2026-11-02T11:22:17.225Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 379,
            "ttftMs": 8988
          },
          {
            "line": 4409,
            "rx": "2026-11-02T11:22:52.039Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 425,
            "ttftMs": 564
          },
          {
            "line": 4544,
            "rx": "2026-11-02T11:23:14.935Z",
            "sessionId": "f1fbb76e-d4fd-459b-8517-56c4c163a40d",
            "source": "first-token-detail",
            "tokensPerSec": 1033,
            "ttftMs": 21567
          },
          {
            "line": 4605,
            "rx": "2026-11-02T11:26:30.959Z",
            "sessionId": "9adfebf2-b744-42b9-ab34-7bc81cc6ee68",
            "source": "first-token-detail",
            "tokensPerSec": 1886,
            "ttftMs": 6294
          },
          {
            "line": 4773,
            "rx": "2026-11-02T11:26:55.881Z",
            "sessionId": "9adfebf2-b744-42b9-ab34-7bc81cc6ee68",
            "source": "first-token-detail",
            "tokensPerSec": 929,
            "ttftMs": 564
          },
          {
            "line": 5031,
            "rx": "2026-11-02T11:27:39.050Z",
            "sessionId": "bd7581ce-61de-4886-a103-5608a9d90d52",
            "source": "first-token-detail",
            "tokensPerSec": 1325,
            "ttftMs": 43169
          },
          {
            "line": 6417,
            "rx": "2026-11-02T11:31:26.089Z",
            "sessionId": "bd7581ce-61de-4886-a103-5608a9d90d52",
            "source": "first-token-detail",
            "tokensPerSec": 366,
            "ttftMs": 435
          },
          {
            "line": 6687,
            "rx": "2026-11-02T11:35:00.968Z",
            "sessionId": "9adfebf2-b744-42b9-ab34-7bc81cc6ee68",
            "source": "first-token-detail",
            "tokensPerSec": 447,
            "ttftMs": 402
          },
          {
            "line": 6785,
            "rx": "2026-11-02T11:35:14.017Z",
            "sessionId": "9adfebf2-b744-42b9-ab34-7bc81cc6ee68",
            "source": "first-token-detail",
            "tokensPerSec": 923,
            "ttftMs": 360
          },
          {
            "line": 6825,
            "rx": "2026-11-02T11:35:18.857Z",
            "sessionId": "9adfebf2-b744-42b9-ab34-7bc81cc6ee68",
            "source": "first-token-detail",
            "tokensPerSec": 689,
            "ttftMs": 302
          },
          {
            "line": 6989,
            "rx": "2026-11-02T11:35:43.621Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 659,
            "ttftMs": 474
          },
          {
            "line": 7250,
            "rx": "2026-11-02T11:36:21.902Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 735,
            "ttftMs": 394
          },
          {
            "line": 7300,
            "rx": "2026-11-02T11:36:38.642Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 1370,
            "ttftMs": 8166
          },
          {
            "line": 7539,
            "rx": "2026-11-02T11:37:38.440Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 1337,
            "ttftMs": 9135
          },
          {
            "line": 7658,
            "rx": "2026-11-02T11:38:08.824Z",
            "sessionId": "8f1d7a7f-69c5-4454-9c09-c0acb3146172",
            "source": "first-token-detail",
            "tokensPerSec": 1325,
            "ttftMs": 9603
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-export-generalize",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-export-generalize",
        "repetition": 3
      },
      "trialId": "craftbook-export-generalize-mlx-gemma4-12b-q4-2026-11-02T12-31-24-072Z-dtnw",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-gemma4-12b-q4-2026-11-02T12-31-24-072Z-dtnw/recording/chat-events.jsonl",
      "recordingSha256": "7746d2550c5615167430eb9b68059a62050928e78bdb612358b026c5d8d6eaa3",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-gemma4-12b-q4-2026-11-02T12-31-24-072Z-dtnw/metrics.json",
      "metricsSha256": "3c23f2613954d94f312a61be1b8808986de7463e3160106fbdec989965b02365",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 999,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 29,
            "rx": "2026-11-02T12:31:45.161Z",
            "sessionId": "ec118226-d004-4cb4-a71c-65c9f0c22bbf",
            "source": "first-token-detail",
            "tokensPerSec": 1091,
            "ttftMs": 19207
          },
          {
            "line": 574,
            "rx": "2026-11-02T12:33:12.344Z",
            "sessionId": "ec118226-d004-4cb4-a71c-65c9f0c22bbf",
            "source": "first-token-detail",
            "tokensPerSec": 907,
            "ttftMs": 413
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-export-generalize",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-export-generalize",
        "repetition": 3
      },
      "trialId": "craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-11-02T12-47-58-690Z-f716",
      "success": true,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-11-02T12-47-58-690Z-f716/recording/chat-events.jsonl",
      "recordingSha256": "58006d7b9e8462fab72e69cebbdf1593eb27c75bdb669375d539eb2c2d86ba65",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-mlx-qwen3-8-27b-q4-2026-11-02T12-47-58-690Z-f716/metrics.json",
      "metricsSha256": "22f08403ab9cbeb7fb151ae9690829daeb8b1d5a297428a3b53274ee12fec1e8",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 483,
        "sampleCount": 18,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 18,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 34,
            "rx": "2026-11-02T12:48:30.103Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 768,
            "ttftMs": 29873
          },
          {
            "line": 4989,
            "rx": "2026-11-02T13:04:32.679Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 542,
            "ttftMs": 9415
          },
          {
            "line": 5105,
            "rx": "2026-11-02T13:04:58.218Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 251,
            "ttftMs": 1411
          },
          {
            "line": 11648,
            "rx": "2026-11-02T13:29:12.460Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 501,
            "ttftMs": 11406
          },
          {
            "line": 11784,
            "rx": "2026-11-02T13:29:33.893Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 214,
            "ttftMs": 1569
          },
          {
            "line": 14597,
            "rx": "2026-11-02T13:40:19.561Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 9023
          },
          {
            "line": 14726,
            "rx": "2026-11-02T13:40:40.772Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 278,
            "ttftMs": 3057
          },
          {
            "line": 15996,
            "rx": "2026-11-02T13:45:42.302Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 465,
            "ttftMs": 7247
          },
          {
            "line": 16128,
            "rx": "2026-11-02T13:46:02.847Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 167,
            "ttftMs": 1367
          },
          {
            "line": 19237,
            "rx": "2026-11-02T13:57:57.038Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 239,
            "ttftMs": 2314
          },
          {
            "line": 21275,
            "rx": "2026-11-02T14:04:48.535Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 1.1,
            "ttftMs": 414378
          },
          {
            "line": 23163,
            "rx": "2026-11-02T14:12:24.322Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 518,
            "ttftMs": 15519
          },
          {
            "line": 23539,
            "rx": "2026-11-02T14:13:32.844Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 11340
          },
          {
            "line": 23782,
            "rx": "2026-11-02T14:14:24.488Z",
            "sessionId": "903ec718-30ae-4ff2-acbd-8dd7374dd2f9",
            "source": "first-token-detail",
            "tokensPerSec": 508,
            "ttftMs": 62486
          },
          {
            "line": 25797,
            "rx": "2026-11-02T14:19:56.911Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 171,
            "ttftMs": 1676
          },
          {
            "line": 27686,
            "rx": "2026-11-02T14:27:03.195Z",
            "sessionId": "903ec718-30ae-4ff2-acbd-8dd7374dd2f9",
            "source": "first-token-detail",
            "tokensPerSec": 507,
            "ttftMs": 380633
          },
          {
            "line": 30310,
            "rx": "2026-11-02T14:36:59.097Z",
            "sessionId": "1d344bfe-fc3f-4e03-b65c-9b289e81a7a6",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 40480
          },
          {
            "line": 30992,
            "rx": "2026-11-02T14:39:26.207Z",
            "sessionId": "9b1d6e94-8605-4d70-acc0-1779a1ed7d3d",
            "source": "first-token-detail",
            "tokensPerSec": 158,
            "ttftMs": 4316
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-author-params",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-params",
        "repetition": 3
      },
      "trialId": "craftbook-author-params-mlx-gemma4-12b-q4-2026-11-02T14-39-55-839Z-slk9",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-gemma4-12b-q4-2026-11-02T14-39-55-839Z-slk9/recording/chat-events.jsonl",
      "recordingSha256": "e5d6c08e1ca89fc1417e807f034d99b38dc427a65f31fc03eebc98edbd10d2f0",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-gemma4-12b-q4-2026-11-02T14-39-55-839Z-slk9/metrics.json",
      "metricsSha256": "87e769f3ab226358d61ad059c536cb00cb3d3377600c07f3950d0e00da50db4b",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 475,
        "sampleCount": 23,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 23,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 31,
            "rx": "2026-11-02T14:40:16.072Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 1077,
            "ttftMs": 18664
          },
          {
            "line": 346,
            "rx": "2026-11-02T14:41:23.940Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 593
          },
          {
            "line": 676,
            "rx": "2026-11-02T14:42:13.716Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 566,
            "ttftMs": 807
          },
          {
            "line": 920,
            "rx": "2026-11-02T14:42:58.453Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 344,
            "ttftMs": 429
          },
          {
            "line": 1039,
            "rx": "2026-11-02T14:43:17.187Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 368,
            "ttftMs": 517
          },
          {
            "line": 1157,
            "rx": "2026-11-02T14:43:35.331Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 372,
            "ttftMs": 539
          },
          {
            "line": 1407,
            "rx": "2026-11-02T14:44:15.122Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 360,
            "ttftMs": 668
          },
          {
            "line": 1532,
            "rx": "2026-11-02T14:44:34.695Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 715,
            "ttftMs": 800
          },
          {
            "line": 1840,
            "rx": "2026-11-02T14:45:29.422Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 400,
            "ttftMs": 537
          },
          {
            "line": 1937,
            "rx": "2026-11-02T14:45:44.929Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 923,
            "ttftMs": 522
          },
          {
            "line": 2110,
            "rx": "2026-11-02T14:46:11.683Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 367,
            "ttftMs": 544
          },
          {
            "line": 2493,
            "rx": "2026-11-02T14:47:10.921Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 447,
            "ttftMs": 640
          },
          {
            "line": 2831,
            "rx": "2026-11-02T14:48:07.306Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 389,
            "ttftMs": 639
          },
          {
            "line": 2953,
            "rx": "2026-11-02T14:48:26.020Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 428,
            "ttftMs": 629
          },
          {
            "line": 3037,
            "rx": "2026-11-02T14:48:41.451Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 996,
            "ttftMs": 731
          },
          {
            "line": 3392,
            "rx": "2026-11-02T14:49:36.955Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 538,
            "ttftMs": 792
          },
          {
            "line": 3568,
            "rx": "2026-11-02T14:50:03.905Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 475,
            "ttftMs": 620
          },
          {
            "line": 3729,
            "rx": "2026-11-02T14:50:28.038Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 416,
            "ttftMs": 633
          },
          {
            "line": 3895,
            "rx": "2026-11-02T14:50:58.104Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 402,
            "ttftMs": 535
          },
          {
            "line": 4112,
            "rx": "2026-11-02T14:51:47.628Z",
            "sessionId": "b7d88ada-5bce-4374-81dc-5d023cf4cecf",
            "source": "first-token-detail",
            "tokensPerSec": 765,
            "ttftMs": 520
          },
          {
            "line": 4596,
            "rx": "2026-11-02T14:56:03.705Z",
            "sessionId": "13a43438-72b4-47a6-9721-42176ad26adc",
            "source": "first-token-detail",
            "tokensPerSec": 1827,
            "ttftMs": 7329
          },
          {
            "line": 4661,
            "rx": "2026-11-02T14:56:15.905Z",
            "sessionId": "13a43438-72b4-47a6-9721-42176ad26adc",
            "source": "first-token-detail",
            "tokensPerSec": 1091,
            "ttftMs": 1471
          },
          {
            "line": 4730,
            "rx": "2026-11-02T14:56:25.671Z",
            "sessionId": "a53be8a1-f21b-4535-8de1-7ef86abfe165",
            "source": "first-token-detail",
            "tokensPerSec": 660,
            "ttftMs": 15047
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-author-params",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-params",
        "repetition": 3
      },
      "trialId": "craftbook-author-params-mlx-qwen3-8-27b-q4-2026-11-02T15-14-24-526Z-k7mx",
      "success": false,
      "includedInCapability": false,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-qwen3-8-27b-q4-2026-11-02T15-14-24-526Z-k7mx/recording/chat-events.jsonl",
      "recordingSha256": "1d7b890e7a2395a2fdba3ce1287f7597d1f71ba1d6a9e0d74096d71310e8ccbb",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-mlx-qwen3-8-27b-q4-2026-11-02T15-14-24-526Z-k7mx/metrics.json",
      "metricsSha256": "c2abb8be34d3c0b701a2b9b754387d92c0cfadc4ab44e756e9c5c7ca1d16c4ed",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 261.5,
        "sampleCount": 8,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 8,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 42,
            "rx": "2026-11-02T15:15:12.318Z",
            "sessionId": "7042b39c-4c94-4816-a659-e84af6bd6559",
            "source": "first-token-detail",
            "tokensPerSec": 457,
            "ttftMs": 46113
          },
          {
            "line": 5118,
            "rx": "2026-11-02T15:32:27.768Z",
            "sessionId": "5fcb2dd5-a379-42f6-af50-1d24b337ad3d",
            "source": "first-token-detail",
            "tokensPerSec": 201,
            "ttftMs": 226049
          },
          {
            "line": 5604,
            "rx": "2026-11-02T15:34:16.083Z",
            "sessionId": "7042b39c-4c94-4816-a659-e84af6bd6559",
            "source": "first-token-detail",
            "tokensPerSec": 220,
            "ttftMs": 2330
          },
          {
            "line": 6016,
            "rx": "2026-11-02T15:35:42.971Z",
            "sessionId": "7042b39c-4c94-4816-a659-e84af6bd6559",
            "source": "first-token-detail",
            "tokensPerSec": 298,
            "ttftMs": 5666
          },
          {
            "line": 13599,
            "rx": "2026-11-02T16:03:50.116Z",
            "sessionId": "7042b39c-4c94-4816-a659-e84af6bd6559",
            "source": "first-token-detail",
            "tokensPerSec": 219,
            "ttftMs": 14994
          },
          {
            "line": 16629,
            "rx": "2026-11-02T16:13:57.967Z",
            "sessionId": "5fcb2dd5-a379-42f6-af50-1d24b337ad3d",
            "source": "first-token-detail",
            "tokensPerSec": 388,
            "ttftMs": 4883
          },
          {
            "line": 18544,
            "rx": "2026-11-02T16:18:40.536Z",
            "sessionId": "5fcb2dd5-a379-42f6-af50-1d24b337ad3d",
            "source": "first-token-detail",
            "tokensPerSec": 225,
            "ttftMs": 1515
          },
          {
            "line": 21802,
            "rx": "2026-11-02T16:26:56.521Z",
            "sessionId": "5fcb2dd5-a379-42f6-af50-1d24b337ad3d",
            "source": "first-token-detail",
            "tokensPerSec": 561,
            "ttftMs": 25276
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-author-gate-script",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-gate-script",
        "repetition": 3
      },
      "trialId": "craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-11-02T18-10-06-901Z-pic2",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-11-02T18-10-06-901Z-pic2/recording/chat-events.jsonl",
      "recordingSha256": "77d1a85416a49ce00631b94ec649f2eb5e2505cd067bdb772f4d02f88e76cf4d",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-qwen3-8-27b-q4-2026-11-02T18-10-06-901Z-pic2/metrics.json",
      "metricsSha256": "b13f9d8263810f7ee730c10576dcb4982897c4494c701a5f057d9ef83d319abb",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 411.5,
        "sampleCount": 2,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 2,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 37,
            "rx": "2026-11-02T18:10:48.373Z",
            "sessionId": "e1bab1f1-2c5c-4b33-93ac-224bd38be24e",
            "source": "first-token-detail",
            "tokensPerSec": 544,
            "ttftMs": 39800
          },
          {
            "line": 2393,
            "rx": "2026-11-02T18:21:36.198Z",
            "sessionId": "e1bab1f1-2c5c-4b33-93ac-224bd38be24e",
            "source": "first-token-detail",
            "tokensPerSec": 279,
            "ttftMs": 16399
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-author-gate-script",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-gate-script",
        "repetition": 3
      },
      "trialId": "craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-11-02T19-08-53-661Z-9l03",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-11-02T19-08-53-661Z-9l03/recording/chat-events.jsonl",
      "recordingSha256": "3ca8c6375acac84172be241a6f656c76622f3f8d557baa4afed1fec275e30aca",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-mlx-gemma4-12b-q4-2026-11-02T19-08-53-661Z-9l03/metrics.json",
      "metricsSha256": "d468659fe78e6428762b702185696b24c76a8be92504a84773b1128692b0d814",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 524.5,
        "sampleCount": 36,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 36,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 32,
            "rx": "2026-11-02T19:09:13.854Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 1122,
            "ttftMs": 18637
          },
          {
            "line": 476,
            "rx": "2026-11-02T19:11:01.538Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 483,
            "ttftMs": 461
          },
          {
            "line": 856,
            "rx": "2026-11-02T19:12:28.169Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 509,
            "ttftMs": 866
          },
          {
            "line": 1261,
            "rx": "2026-11-02T19:14:03.775Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 504,
            "ttftMs": 801
          },
          {
            "line": 1715,
            "rx": "2026-11-02T19:15:26.940Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 486,
            "ttftMs": 812
          },
          {
            "line": 1920,
            "rx": "2026-11-02T19:16:10.854Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 376,
            "ttftMs": 720
          },
          {
            "line": 2159,
            "rx": "2026-11-02T19:16:59.628Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 380,
            "ttftMs": 568
          },
          {
            "line": 2266,
            "rx": "2026-11-02T19:17:15.727Z",
            "sessionId": "3a5e5376-26e6-4b66-9fec-f3eee1b63c43",
            "source": "first-token-detail",
            "tokensPerSec": 523,
            "ttftMs": 1003
          },
          {
            "line": 2804,
            "rx": "2026-11-02T19:21:59.395Z",
            "sessionId": "69a5bbd6-7da8-46ca-82c6-c8e711bc7680",
            "source": "first-token-detail",
            "tokensPerSec": 1842,
            "ttftMs": 7261
          },
          {
            "line": 2868,
            "rx": "2026-11-02T19:22:11.804Z",
            "sessionId": "69a5bbd6-7da8-46ca-82c6-c8e711bc7680",
            "source": "first-token-detail",
            "tokensPerSec": 978,
            "ttftMs": 1289
          },
          {
            "line": 2977,
            "rx": "2026-11-02T19:22:27.089Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 482,
            "ttftMs": 20493
          },
          {
            "line": 3160,
            "rx": "2026-11-02T19:22:47.646Z",
            "sessionId": "69a5bbd6-7da8-46ca-82c6-c8e711bc7680",
            "source": "first-token-detail",
            "tokensPerSec": 800,
            "ttftMs": 625
          },
          {
            "line": 3266,
            "rx": "2026-11-02T19:23:01.286Z",
            "sessionId": "69a5bbd6-7da8-46ca-82c6-c8e711bc7680",
            "source": "first-token-detail",
            "tokensPerSec": 690,
            "ttftMs": 505
          },
          {
            "line": 3414,
            "rx": "2026-11-02T19:23:22.542Z",
            "sessionId": "69a5bbd6-7da8-46ca-82c6-c8e711bc7680",
            "source": "first-token-detail",
            "tokensPerSec": 526,
            "ttftMs": 433
          },
          {
            "line": 3745,
            "rx": "2026-11-02T19:24:09.116Z",
            "sessionId": "69a5bbd6-7da8-46ca-82c6-c8e711bc7680",
            "source": "first-token-detail",
            "tokensPerSec": 546,
            "ttftMs": 548
          },
          {
            "line": 3902,
            "rx": "2026-11-02T19:24:40.008Z",
            "sessionId": "44fcdf47-a9af-405f-9436-089af17448bd",
            "source": "first-token-detail",
            "tokensPerSec": 1279,
            "ttftMs": 9710
          },
          {
            "line": 3985,
            "rx": "2026-11-02T19:24:56.606Z",
            "sessionId": "44fcdf47-a9af-405f-9436-089af17448bd",
            "source": "first-token-detail",
            "tokensPerSec": 564,
            "ttftMs": 424
          },
          {
            "line": 4175,
            "rx": "2026-11-02T19:25:25.328Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 589,
            "ttftMs": 608
          },
          {
            "line": 4249,
            "rx": "2026-11-02T19:25:35.876Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 403,
            "ttftMs": 403
          },
          {
            "line": 4409,
            "rx": "2026-11-02T19:26:01.711Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 284,
            "ttftMs": 416
          },
          {
            "line": 4863,
            "rx": "2026-11-02T19:27:13.656Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 471,
            "ttftMs": 617
          },
          {
            "line": 4970,
            "rx": "2026-11-02T19:27:34.890Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 1003,
            "ttftMs": 3760
          },
          {
            "line": 5014,
            "rx": "2026-11-02T19:27:46.853Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 486,
            "ttftMs": 660
          },
          {
            "line": 5170,
            "rx": "2026-11-02T19:28:11.549Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 434,
            "ttftMs": 695
          },
          {
            "line": 5627,
            "rx": "2026-11-02T19:29:23.760Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 267,
            "ttftMs": 519
          },
          {
            "line": 5846,
            "rx": "2026-11-02T19:30:09.458Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 997,
            "ttftMs": 12161
          },
          {
            "line": 5919,
            "rx": "2026-11-02T19:30:21.105Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 516,
            "ttftMs": 1004
          },
          {
            "line": 6035,
            "rx": "2026-11-02T19:30:38.691Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 373,
            "ttftMs": 513
          },
          {
            "line": 6218,
            "rx": "2026-11-02T19:31:07.925Z",
            "sessionId": "f93fb8fa-4434-4b35-a66a-be3d415162a6",
            "source": "first-token-detail",
            "tokensPerSec": 362,
            "ttftMs": 472
          },
          {
            "line": 6314,
            "rx": "2026-11-02T19:31:28.609Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 1362,
            "ttftMs": 3894
          },
          {
            "line": 6338,
            "rx": "2026-11-02T19:31:40.003Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 1267,
            "ttftMs": 5329
          },
          {
            "line": 6366,
            "rx": "2026-11-02T19:31:56.866Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 1289,
            "ttftMs": 7062
          },
          {
            "line": 6472,
            "rx": "2026-11-02T19:32:32.088Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 1098,
            "ttftMs": 7028
          },
          {
            "line": 6505,
            "rx": "2026-11-02T19:32:48.724Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 1091,
            "ttftMs": 8556
          },
          {
            "line": 6575,
            "rx": "2026-11-02T19:33:19.754Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 983,
            "ttftMs": 17098
          },
          {
            "line": 6757,
            "rx": "2026-11-02T19:33:51.204Z",
            "sessionId": "424ad1e0-7484-43ce-9b3a-8d74e0d11576",
            "source": "first-token-detail",
            "tokensPerSec": 457,
            "ttftMs": 382
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-qwen-mlx-complex-work-craftbook-author-fanout",
        "provider": "mlx",
        "model": "qwen3.8-27b-q4",
        "targetId": "qwen-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-fanout",
        "repetition": 3
      },
      "trialId": "craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-11-02T19-35-24-686Z-nnsw",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-11-02T19-35-24-686Z-nnsw/recording/chat-events.jsonl",
      "recordingSha256": "c53cb9fb6839d462421c57dee794acac424cbafa5c78b15335e3bdc52d79b69f",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-qwen-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-qwen3-8-27b-q4-2026-11-02T19-35-24-686Z-nnsw/metrics.json",
      "metricsSha256": "8cfaa74049b61a3be6a82adcf84d7939564001fc5cad01071c198872d1900bb7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 543,
        "sampleCount": 1,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 1,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 37,
            "rx": "2026-11-02T19:36:06.295Z",
            "sessionId": "622e669b-d259-4440-9922-bab9a5a48274",
            "source": "first-token-detail",
            "tokensPerSec": 543,
            "ttftMs": 39838
          }
        ]
      }
    },
    {
      "cell": {
        "id": "r3-gemma-mlx-complex-work-craftbook-author-fanout",
        "provider": "mlx",
        "model": "gemma4-12b-q4",
        "targetId": "gemma-mlx",
        "suite": "complex-work",
        "scenario": "craftbook-author-fanout",
        "repetition": 3
      },
      "trialId": "craftbook-author-fanout-mlx-gemma4-12b-q4-2026-11-02T21-03-44-160Z-lsom",
      "success": false,
      "includedInCapability": true,
      "recording": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-gemma4-12b-q4-2026-11-02T21-03-44-160Z-lsom/recording/chat-events.jsonl",
      "recordingSha256": "8484d2016650ebcb19d66db2ce324ec5f95eed863887be5de3cc8b8587ce814c",
      "metrics": "evals/runs/scorecard-2026-10-27-m5max-native-0.1.54/r3-gemma-mlx-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-mlx-gemma4-12b-q4-2026-11-02T21-03-44-160Z-lsom/metrics.json",
      "metricsSha256": "2bbcc0d37ec71a63f617a706fa62647264cf4a5dc0e0453d5dd60c87e46dd5d7",
      "originalAggregatePrefillTokensPerSec": null,
      "purePrefillDurationMs": null,
      "recovered": {
        "source": "first-token-detail",
        "aggregation": "median-observed-events",
        "tokensPerSec": 506,
        "sampleCount": 21,
        "structuredSampleCount": 0,
        "firstTokenSampleCount": 21,
        "malformedLineCount": 0,
        "duplicateLineCount": 0,
        "observations": [
          {
            "line": 30,
            "rx": "2026-11-02T21:04:05.393Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 1040,
            "ttftMs": 19562
          },
          {
            "line": 321,
            "rx": "2026-11-02T21:05:06.700Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 352,
            "ttftMs": 472
          },
          {
            "line": 402,
            "rx": "2026-11-02T21:05:18.310Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 370,
            "ttftMs": 462
          },
          {
            "line": 517,
            "rx": "2026-11-02T21:05:35.729Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 570,
            "ttftMs": 862
          },
          {
            "line": 951,
            "rx": "2026-11-02T21:06:57.009Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 326,
            "ttftMs": 516
          },
          {
            "line": 1016,
            "rx": "2026-11-02T21:07:06.514Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 399,
            "ttftMs": 569
          },
          {
            "line": 1219,
            "rx": "2026-11-02T21:07:43.272Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 1032,
            "ttftMs": 564
          },
          {
            "line": 1648,
            "rx": "2026-11-02T21:08:59.043Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 910,
            "ttftMs": 634
          },
          {
            "line": 2150,
            "rx": "2026-11-02T21:10:25.702Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 390,
            "ttftMs": 594
          },
          {
            "line": 2355,
            "rx": "2026-11-02T21:10:57.094Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 364,
            "ttftMs": 665
          },
          {
            "line": 2578,
            "rx": "2026-11-02T21:11:31.985Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 466,
            "ttftMs": 789
          },
          {
            "line": 2808,
            "rx": "2026-11-02T21:12:07.605Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 386,
            "ttftMs": 691
          },
          {
            "line": 3082,
            "rx": "2026-11-02T21:12:51.470Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 921,
            "ttftMs": 630
          },
          {
            "line": 3481,
            "rx": "2026-11-02T21:14:02.000Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 356,
            "ttftMs": 590
          },
          {
            "line": 3813,
            "rx": "2026-11-02T21:15:04.442Z",
            "sessionId": "542bc823-c9c8-4d71-90fe-529063100ff5",
            "source": "first-token-detail",
            "tokensPerSec": 1038,
            "ttftMs": 5264
          },
          {
            "line": 4008,
            "rx": "2026-11-02T21:15:39.014Z",
            "sessionId": "cc8cdafd-35ee-4129-9f9c-367cb2119c76",
            "source": "first-token-detail",
            "tokensPerSec": 506,
            "ttftMs": 1145
          },
          {
            "line": 4074,
            "rx": "2026-11-02T21:15:52.991Z",
            "sessionId": "542bc823-c9c8-4d71-90fe-529063100ff5",
            "source": "first-token-detail",
            "tokensPerSec": 987,
            "ttftMs": 5615
          },
          {
            "line": 4179,
            "rx": "2026-11-02T21:16:35.020Z",
            "sessionId": "542bc823-c9c8-4d71-90fe-529063100ff5",
            "source": "first-token-detail",
            "tokensPerSec": 979,
            "ttftMs": 7322
          },
          {
            "line": 4602,
            "rx": "2026-11-02T21:17:44.437Z",
            "sessionId": "542bc823-c9c8-4d71-90fe-529063100ff5",
            "source": "first-token-detail",
            "tokensPerSec": 489,
            "ttftMs": 417
          },
          {
            "line": 4936,
            "rx": "2026-11-02T21:18:46.818Z",
            "sessionId": "542bc823-c9c8-4d71-90fe-529063100ff5",
            "source": "first-token-detail",
            "tokensPerSec": 1164,
            "ttftMs": 7003
          },
          {
            "line": 4966,
            "rx": "2026-11-02T21:19:06.997Z",
            "sessionId": "542bc823-c9c8-4d71-90fe-529063100ff5",
            "source": "first-token-detail",
            "tokensPerSec": 1038,
            "ttftMs": 7577
          }
        ]
      }
    }
  ]
}