{
  "runs": {
    "qwen3-4b-s17": {
      "complete": true,
      "model": "Qwen/Qwen3-4B-Instruct-2507",
      "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 43,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "apparati di nuova generazione",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "comportamenti spontanei",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "crescita di capitale",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fatturato di valore",
              "correct": false
            }
          ],
          "logprob": -13.21737003326416,
          "logprob_with_eos": -20.53278923034668,
          "rank": {
            "rank": 565,
            "rank_pessimistic": 565,
            "near_tie_rank_interval": [
              559,
              570
            ]
          },
          "prefix_rank": {
            "rank": 42,
            "rank_pessimistic": 42,
            "near_tie_rank_interval": [
              42,
              42
            ]
          },
          "numerical_check": {
            "cached_single": [
              -20.59531593322754,
              -13.199804306030273,
              -7.395512580871582
            ],
            "uncached_single": [
              -20.87639617919922,
              -13.267451286315918,
              -7.608944892883301
            ],
            "cached_batch": [
              -20.53278923034668,
              -13.21737003326416,
              -7.315419673919678
            ],
            "cache_vs_full_max_error": 0.2810802459716797
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 37,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "half-life",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "time to go",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "half-life",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "time to go",
              "correct": false
            }
          ],
          "logprob": -10.57080364227295,
          "logprob_with_eos": -10.574780464172363,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.560571670532227,
              -10.556584358215332,
              -0.003987814299762249
            ],
            "uncached_single": [
              -10.601180076599121,
              -10.596920013427734,
              -0.004260511137545109
            ],
            "cached_batch": [
              -10.574780464172363,
              -10.57080364227295,
              -0.003976890817284584
            ],
            "cache_vs_full_max_error": 0.04060840606689453
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 46,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "appoggio di terra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "appoggio a terra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "struttura di sostegno",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenta di legno",
              "correct": false
            }
          ],
          "logprob": -18.948640823364258,
          "logprob_with_eos": -18.949312210083008,
          "rank": {
            "rank": 306,
            "rank_pessimistic": 306,
            "near_tie_rank_interval": [
              301,
              312
            ]
          },
          "prefix_rank": {
            "rank": 309,
            "rank_pessimistic": 309,
            "near_tie_rank_interval": [
              303,
              316
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.89986801147461,
              -18.89910125732422,
              -0.0007681279676035047
            ],
            "uncached_single": [
              -18.885705947875977,
              -18.88492202758789,
              -0.000783732277341187
            ],
            "cached_batch": [
              -18.949312210083008,
              -18.948640823364258,
              -0.0006708040600642562
            ],
            "cache_vs_full_max_error": 0.014179229736328125
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 48,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "tacito rispetto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "tasto rosso",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "vago e tondo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "senza fiato",
              "correct": false
            }
          ],
          "logprob": -12.587839126586914,
          "logprob_with_eos": -12.588342666625977,
          "rank": {
            "rank": 40,
            "rank_pessimistic": 40,
            "near_tie_rank_interval": [
              40,
              40
            ]
          },
          "prefix_rank": {
            "rank": 40,
            "rank_pessimistic": 40,
            "near_tie_rank_interval": [
              40,
              41
            ]
          },
          "numerical_check": {
            "cached_single": [
              -12.551857948303223,
              -12.55141544342041,
              -0.00044204952428117394
            ],
            "uncached_single": [
              -12.571208953857422,
              -12.570707321166992,
              -0.0005017452058382332
            ],
            "cached_batch": [
              -12.588342666625977,
              -12.587839126586914,
              -0.0005036516231484711
            ],
            "cache_vs_full_max_error": 0.01935100555419922
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 66,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta figura in campo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una pessima notizia",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brava maestra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta faccia in pubblico",
              "correct": false
            }
          ],
          "logprob": -8.486196517944336,
          "logprob_with_eos": -8.522729873657227,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "numerical_check": {
            "cached_single": [
              -8.558725357055664,
              -8.525700569152832,
              -0.03302453085780144
            ],
            "uncached_single": [
              -8.514036178588867,
              -8.48150634765625,
              -0.03253030404448509
            ],
            "cached_batch": [
              -8.522729873657227,
              -8.486196517944336,
              -0.03653394430875778
            ],
            "cache_vs_full_max_error": 0.044689178466796875
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 187,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "manifestazioni di potere",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "manifestazioni di potere",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "fenomeni di scala",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "manifestazioni di potere",
              "correct": false
            }
          ],
          "logprob": -13.518392562866211,
          "logprob_with_eos": -28.07032012939453,
          "rank": {
            "rank": 1392,
            "rank_pessimistic": 1392,
            "near_tie_rank_interval": [
              1387,
              1397
            ]
          },
          "prefix_rank": {
            "rank": 41,
            "rank_pessimistic": 41,
            "near_tie_rank_interval": [
              40,
              42
            ]
          },
          "numerical_check": {
            "cached_single": [
              -28.171844482421875,
              -13.60799503326416,
              -14.563848495483398
            ],
            "uncached_single": [
              -28.135330200195312,
              -13.530328750610352,
              -14.604999542236328
            ],
            "cached_batch": [
              -28.07032012939453,
              -13.518392562866211,
              -14.551926612854004
            ],
            "cache_vs_full_max_error": 0.0776662826538086
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 181,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "\"the slow decay\"",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "time to go",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "half-life period",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fate's wheel turns",
              "correct": false
            }
          ],
          "logprob": -11.475361824035645,
          "logprob_with_eos": -11.491796493530273,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "numerical_check": {
            "cached_single": [
              -11.430076599121094,
              -11.415002822875977,
              -0.015073326416313648
            ],
            "uncached_single": [
              -11.45152759552002,
              -11.436087608337402,
              -0.015439862385392189
            ],
            "cached_batch": [
              -11.491796493530273,
              -11.475361824035645,
              -0.01643524318933487
            ],
            "cache_vs_full_max_error": 0.02145099639892578
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 190,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "appoggiate in mare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "appoggiate a terra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "struttura di sostegno",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenta di legno",
              "correct": false
            }
          ],
          "logprob": -18.4717960357666,
          "logprob_with_eos": -18.476089477539062,
          "rank": {
            "rank": 294,
            "rank_pessimistic": 294,
            "near_tie_rank_interval": [
              290,
              299
            ]
          },
          "prefix_rank": {
            "rank": 302,
            "rank_pessimistic": 302,
            "near_tie_rank_interval": [
              298,
              308
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.6058406829834,
              -18.6016788482666,
              -0.004161033313721418
            ],
            "uncached_single": [
              -18.443214416503906,
              -18.439470291137695,
              -0.0037443782202899456
            ],
            "cached_batch": [
              -18.476089477539062,
              -18.4717960357666,
              -0.0042930347844958305
            ],
            "cache_vs_full_max_error": 0.1626262664794922
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 192,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "\"Non posso dire\"",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "tasto rosso",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "canto inascoltato",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "senza fiato",
              "correct": false
            }
          ],
          "logprob": -13.59853744506836,
          "logprob_with_eos": -13.598918914794922,
          "rank": {
            "rank": 73,
            "rank_pessimistic": 73,
            "near_tie_rank_interval": [
              71,
              75
            ]
          },
          "prefix_rank": {
            "rank": 73,
            "rank_pessimistic": 73,
            "near_tie_rank_interval": [
              71,
              75
            ]
          },
          "numerical_check": {
            "cached_single": [
              -13.692562103271484,
              -13.692184448242188,
              -0.0003780603874474764
            ],
            "uncached_single": [
              -13.569849014282227,
              -13.569425582885742,
              -0.00042226933874189854
            ],
            "cached_batch": [
              -13.598918914794922,
              -13.59853744506836,
              -0.00038223114097490907
            ],
            "cache_vs_full_max_error": 0.12275886535644531
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 210,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta figura",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una pessima notizia",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brava maestra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta faccia a sinistra",
              "correct": false
            }
          ],
          "logprob": -9.189382553100586,
          "logprob_with_eos": -9.198657989501953,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "numerical_check": {
            "cached_single": [
              -9.3528470993042,
              -9.343728065490723,
              -0.009119288064539433
            ],
            "uncached_single": [
              -9.208534240722656,
              -9.199235916137695,
              -0.009298352524638176
            ],
            "cached_batch": [
              -9.198657989501953,
              -9.189382553100586,
              -0.009276267141103745
            ],
            "cache_vs_full_max_error": 0.14449214935302734
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 40,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 73,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.4,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": true,
      "adapter_seed": 17,
      "adapter_sha256": {
        "README.md": "678742275e6a4d4a361eb467bc11c04c21794cc63a67e7f0f78b989dfd6d6bfb",
        "adapter_config.json": "2a845a051d0a7ee74171e1de07216e3fa40141c4795ea1ce515fc3079ae63ace",
        "adapter_model.safetensors": "aa1580835b949304abb741b7d27a9dfae18bcd87b60413b290a4aa1dfda3c875",
        "chat_template.jinja": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
        "tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
        "tokenizer_config.json": "e7979df876a7bccca3d4300c252ba4a13657c7ac23e62a98498c8b56dfac39ab"
      },
      "prompt_source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 219.979430049,
      "gpu": "NVIDIA A100-SXM4-80GB",
      "source_sha256": "40036973521582e377746b371583eb3e66ab6dbe8671ea5e03179fe37c54d447",
      "plan_sha256": "ae270c72f21785f2396a35f09d83cc62b1f7f495d5fe690ef633083e96112239",
      "protocol_sha256": "12d5d8c26e2557fd1e4d255ae478de508b7eaaea3462b3a4f2673086d9fd3b98",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen3-4b-s29": {
      "complete": true,
      "model": "Qwen/Qwen3-4B-Instruct-2507",
      "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 43,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "apparaturi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "attivismo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "attualità",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "appariti di fronte",
              "correct": false
            }
          ],
          "logprob": -13.078164100646973,
          "logprob_with_eos": -14.011289596557617,
          "rank": {
            "rank": 59,
            "rank_pessimistic": 59,
            "near_tie_rank_interval": [
              57,
              59
            ]
          },
          "prefix_rank": {
            "rank": 39,
            "rank_pessimistic": 39,
            "near_tie_rank_interval": [
              37,
              39
            ]
          },
          "numerical_check": {
            "cached_single": [
              -13.938459396362305,
              -12.94912338256836,
              -0.9893365502357483
            ],
            "uncached_single": [
              -14.023157119750977,
              -13.072559356689453,
              -0.9505980014801025
            ],
            "cached_batch": [
              -14.011289596557617,
              -13.078164100646973,
              -0.9331260919570923
            ],
            "cache_vs_full_max_error": 0.12343597412109375
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 37,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "half-life series",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "alpha chain",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "alpha chain",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fate's long chain",
              "correct": false
            }
          ],
          "logprob": -10.80994987487793,
          "logprob_with_eos": -10.820030212402344,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.788044929504395,
              -10.778493881225586,
              -0.009551186114549637
            ],
            "uncached_single": [
              -10.885066986083984,
              -10.87753963470459,
              -0.00752743985503912
            ],
            "cached_batch": [
              -10.820030212402344,
              -10.80994987487793,
              -0.010080852545797825
            ],
            "cache_vs_full_max_error": 0.0990457534790039
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 46,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "appoggio di fondo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "appoggiate a terra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "struttura di base",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenta di legno",
              "correct": false
            }
          ],
          "logprob": -19.40478515625,
          "logprob_with_eos": -19.4060115814209,
          "rank": {
            "rank": 372,
            "rank_pessimistic": 372,
            "near_tie_rank_interval": [
              367,
              378
            ]
          },
          "prefix_rank": {
            "rank": 376,
            "rank_pessimistic": 376,
            "near_tie_rank_interval": [
              372,
              382
            ]
          },
          "numerical_check": {
            "cached_single": [
              -19.509950637817383,
              -19.50876235961914,
              -0.0011893587652593851
            ],
            "uncached_single": [
              -19.469127655029297,
              -19.467954635620117,
              -0.0011741180205717683
            ],
            "cached_batch": [
              -19.4060115814209,
              -19.40478515625,
              -0.0012259118957445025
            ],
            "cache_vs_full_max_error": 0.04082298278808594
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 48,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "tasto \"bloccato\"",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "tasto rosso",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "canto terzo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "freno a mano",
              "correct": false
            }
          ],
          "logprob": -11.948569297790527,
          "logprob_with_eos": -11.948822021484375,
          "rank": {
            "rank": 31,
            "rank_pessimistic": 31,
            "near_tie_rank_interval": [
              30,
              31
            ]
          },
          "prefix_rank": {
            "rank": 31,
            "rank_pessimistic": 31,
            "near_tie_rank_interval": [
              30,
              31
            ]
          },
          "numerical_check": {
            "cached_single": [
              -11.94259262084961,
              -11.942333221435547,
              -0.000259723310591653
            ],
            "uncached_single": [
              -12.125381469726562,
              -12.125120162963867,
              -0.0002611534437164664
            ],
            "cached_batch": [
              -11.948822021484375,
              -11.948569297790527,
              -0.0002520958660170436
            ],
            "cache_vs_full_max_error": 0.18278884887695312
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 66,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta figura fa la notizia",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una brutta faccia in pubblico",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brava maestra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta faccia in pubblico",
              "correct": false
            }
          ],
          "logprob": -7.722171306610107,
          "logprob_with_eos": -7.762502670288086,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "numerical_check": {
            "cached_single": [
              -7.8542070388793945,
              -7.818429470062256,
              -0.035777747631073
            ],
            "uncached_single": [
              -7.721880912780762,
              -7.683772087097168,
              -0.038109004497528076
            ],
            "cached_batch": [
              -7.762502670288086,
              -7.722171306610107,
              -0.04033168405294418
            ],
            "cache_vs_full_max_error": 0.1346573829650879
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 187,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "apparati di rilievo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "apparimenti di proprietà",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "manifestazioni di potere",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fatturazione di valore",
              "correct": false
            }
          ],
          "logprob": -12.887310981750488,
          "logprob_with_eos": -27.12250518798828,
          "rank": {
            "rank": 1337,
            "rank_pessimistic": 1337,
            "near_tie_rank_interval": [
              1336,
              1341
            ]
          },
          "prefix_rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              25,
              26
            ]
          },
          "numerical_check": {
            "cached_single": [
              -26.902692794799805,
              -12.807275772094727,
              -14.095417022705078
            ],
            "uncached_single": [
              -27.011863708496094,
              -12.887971878051758,
              -14.123889923095703
            ],
            "cached_batch": [
              -27.12250518798828,
              -12.887310981750488,
              -14.235194206237793
            ],
            "cache_vs_full_max_error": 0.10917091369628906
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 181,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "lead to the end",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "lead to a conclusion",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "alpha emission",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fate's fall to the ground",
              "correct": false
            }
          ],
          "logprob": -10.236251831054688,
          "logprob_with_eos": -10.5535249710083,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.545221328735352,
              -10.243452072143555,
              -0.3017699420452118
            ],
            "uncached_single": [
              -10.611401557922363,
              -10.316817283630371,
              -0.29458433389663696
            ],
            "cached_batch": [
              -10.5535249710083,
              -10.236251831054688,
              -0.3172730803489685
            ],
            "cache_vs_full_max_error": 0.0733652114868164
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 190,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "appoggio di fondo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "appoggiate a terra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "struttura di sostegno",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenta di legno",
              "correct": false
            }
          ],
          "logprob": -18.351726531982422,
          "logprob_with_eos": -18.360061645507812,
          "rank": {
            "rank": 334,
            "rank_pessimistic": 334,
            "near_tie_rank_interval": [
              328,
              341
            ]
          },
          "prefix_rank": {
            "rank": 360,
            "rank_pessimistic": 360,
            "near_tie_rank_interval": [
              352,
              366
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.470186233520508,
              -18.461835861206055,
              -0.008349984884262085
            ],
            "uncached_single": [
              -18.37500762939453,
              -18.366762161254883,
              -0.008245119825005531
            ],
            "cached_batch": [
              -18.360061645507812,
              -18.351726531982422,
              -0.00833804439753294
            ],
            "cache_vs_full_max_error": 0.09517860412597656
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 192,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "tacito rispetto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "tacito verbale",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "canto e prosa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "senza fiato",
              "correct": false
            }
          ],
          "logprob": -12.654473304748535,
          "logprob_with_eos": -12.65488338470459,
          "rank": {
            "rank": 56,
            "rank_pessimistic": 56,
            "near_tie_rank_interval": [
              55,
              57
            ]
          },
          "prefix_rank": {
            "rank": 56,
            "rank_pessimistic": 56,
            "near_tie_rank_interval": [
              55,
              57
            ]
          },
          "numerical_check": {
            "cached_single": [
              -12.613656997680664,
              -12.613288879394531,
              -0.0003670972364488989
            ],
            "uncached_single": [
              -12.656803131103516,
              -12.656436920166016,
              -0.0003666205739136785
            ],
            "cached_batch": [
              -12.65488338470459,
              -12.654473304748535,
              -0.0004107108688913286
            ],
            "cache_vs_full_max_error": 0.043148040771484375
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 210,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta figura",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una brutta faccia a sinistra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brava maestra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta faccia a sinistra",
              "correct": false
            }
          ],
          "logprob": -9.248835563659668,
          "logprob_with_eos": -9.267193794250488,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "numerical_check": {
            "cached_single": [
              -9.324665069580078,
              -9.304540634155273,
              -0.02012430503964424
            ],
            "uncached_single": [
              -9.295438766479492,
              -9.279312133789062,
              -0.01612725481390953
            ],
            "cached_batch": [
              -9.267193794250488,
              -9.248835563659668,
              -0.018358316272497177
            ],
            "cache_vs_full_max_error": 0.029226303100585938
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 31,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.6,
            "100": 0.8,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 56,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.4,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": true,
      "adapter_seed": 29,
      "adapter_sha256": {
        "README.md": "678742275e6a4d4a361eb467bc11c04c21794cc63a67e7f0f78b989dfd6d6bfb",
        "adapter_config.json": "96d1bb00f3928ee52144d5e4f688d98f4223c3fa577b365b043a8ea70d34f401",
        "adapter_model.safetensors": "6352416d68a4e050e2a7e532b515581f20a8067b28ef83d0efb6dd74d2b33208",
        "chat_template.jinja": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
        "tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
        "tokenizer_config.json": "e7979df876a7bccca3d4300c252ba4a13657c7ac23e62a98498c8b56dfac39ab"
      },
      "prompt_source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 106.866995619,
      "gpu": "NVIDIA A100-SXM4-80GB",
      "source_sha256": "40036973521582e377746b371583eb3e66ab6dbe8671ea5e03179fe37c54d447",
      "plan_sha256": "ae270c72f21785f2396a35f09d83cc62b1f7f495d5fe690ef633083e96112239",
      "protocol_sha256": "12d5d8c26e2557fd1e4d255ae478de508b7eaaea3462b3a4f2673086d9fd3b98",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen35-9b-s17": {
      "complete": true,
      "model": "Qwen/Qwen3.5-9B",
      "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 44,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "successi personali",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "costituzioni nuove",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "successi personali",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "risultati inediti",
              "correct": false
            }
          ],
          "logprob": -16.461288452148438,
          "logprob_with_eos": -22.737043380737305,
          "rank": {
            "rank": 856,
            "rank_pessimistic": 856,
            "near_tie_rank_interval": [
              852,
              864
            ]
          },
          "prefix_rank": {
            "rank": 245,
            "rank_pessimistic": 245,
            "near_tie_rank_interval": [
              243,
              246
            ]
          },
          "numerical_check": {
            "cached_single": [
              -22.753494262695312,
              -16.53900146484375,
              -6.214494705200195
            ],
            "uncached_single": [
              -22.81612777709961,
              -16.60685157775879,
              -6.20927619934082
            ],
            "cached_batch": [
              -22.737043380737305,
              -16.461288452148438,
              -6.275755882263184
            ],
            "cache_vs_full_max_error": 0.06785011291503906
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 42,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "process of elimination",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "process of elimination",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "prime number series",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fission of atoms",
              "correct": false
            }
          ],
          "logprob": -11.561575889587402,
          "logprob_with_eos": -12.292689323425293,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -12.499391555786133,
              -11.689924240112305,
              -0.8094671964645386
            ],
            "uncached_single": [
              -12.432515144348145,
              -11.697683334350586,
              -0.7348314523696899
            ],
            "cached_batch": [
              -12.292689323425293,
              -11.561575889587402,
              -0.7311134338378906
            ],
            "cache_vs_full_max_error": 0.07463574409484863
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 47,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "abitazioni basse",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "costruzioni moderne",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "abitanti di stive",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "stabilimenti balneari",
              "correct": false
            }
          ],
          "logprob": -15.25957202911377,
          "logprob_with_eos": -15.259625434875488,
          "rank": {
            "rank": 146,
            "rank_pessimistic": 146,
            "near_tie_rank_interval": [
              145,
              150
            ]
          },
          "prefix_rank": {
            "rank": 150,
            "rank_pessimistic": 150,
            "near_tie_rank_interval": [
              147,
              154
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.261070251464844,
              -15.261018753051758,
              -5.1377883210079744e-05
            ],
            "uncached_single": [
              -15.275964736938477,
              -15.27591323852539,
              -5.054346183896996e-05
            ],
            "cached_batch": [
              -15.259625434875488,
              -15.25957202911377,
              -5.2927523938706145e-05
            ],
            "cache_vs_full_max_error": 0.014894485473632812
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 49,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "taccio dico",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "taccio corto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "taccio fitto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "taccio fitto",
              "correct": false
            }
          ],
          "logprob": -9.86646556854248,
          "logprob_with_eos": -9.870713233947754,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              3
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "numerical_check": {
            "cached_single": [
              -9.848841667175293,
              -9.845091819763184,
              -0.0037497226148843765
            ],
            "uncached_single": [
              -9.863809585571289,
              -9.859573364257812,
              -0.004236532840877771
            ],
            "cached_batch": [
              -9.870713233947754,
              -9.86646556854248,
              -0.004247572273015976
            ],
            "cache_vs_full_max_error": 0.014967918395996094
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 64,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta figura in giro",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "un brutto pezzo di roba",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brutta figura in mano",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fata malinconica del mare",
              "correct": false
            }
          ],
          "logprob": -11.025594711303711,
          "logprob_with_eos": -12.111085891723633,
          "rank": {
            "rank": 15,
            "rank_pessimistic": 15,
            "near_tie_rank_interval": [
              15,
              15
            ]
          },
          "prefix_rank": {
            "rank": 26,
            "rank_pessimistic": 26,
            "near_tie_rank_interval": [
              25,
              26
            ]
          },
          "numerical_check": {
            "cached_single": [
              -11.966574668884277,
              -10.99708366394043,
              -0.9694904088973999
            ],
            "uncached_single": [
              -12.099066734313965,
              -11.057870864868164,
              -1.041196346282959
            ],
            "cached_batch": [
              -12.111085891723633,
              -11.025594711303711,
              -1.0854909420013428
            ],
            "cache_vs_full_max_error": 0.1324920654296875
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 152,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "risultati aggregati",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "costituzione italiana",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "risultati inattesi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "risultati aggregati",
              "correct": false
            }
          ],
          "logprob": -18.62497329711914,
          "logprob_with_eos": -25.96381378173828,
          "rank": {
            "rank": 1301,
            "rank_pessimistic": 1301,
            "near_tie_rank_interval": [
              1295,
              1307
            ]
          },
          "prefix_rank": {
            "rank": 510,
            "rank_pessimistic": 510,
            "near_tie_rank_interval": [
              506,
              518
            ]
          },
          "numerical_check": {
            "cached_single": [
              -25.91722297668457,
              -18.569185256958008,
              -7.348038196563721
            ],
            "uncached_single": [
              -25.885162353515625,
              -18.558155059814453,
              -7.327008247375488
            ],
            "cached_batch": [
              -25.96381378173828,
              -18.62497329711914,
              -7.338841915130615
            ],
            "cache_vs_full_max_error": 0.03206062316894531
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 150,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "change of elements",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "change of heart",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "radioactive element",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fissione di elementi",
              "correct": false
            }
          ],
          "logprob": -10.268878936767578,
          "logprob_with_eos": -10.716479301452637,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.812817573547363,
              -10.313935279846191,
              -0.49888211488723755
            ],
            "uncached_single": [
              -10.752172470092773,
              -10.310550689697266,
              -0.44162076711654663
            ],
            "cached_batch": [
              -10.716479301452637,
              -10.268878936767578,
              -0.4476006329059601
            ],
            "cache_vs_full_max_error": 0.060645103454589844
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 155,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "abitazioni di lusso",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "costruzioni moderne",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "abitanti delle paludi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "stabilimenti balneari",
              "correct": false
            }
          ],
          "logprob": -16.237628936767578,
          "logprob_with_eos": -16.23769760131836,
          "rank": {
            "rank": 270,
            "rank_pessimistic": 270,
            "near_tie_rank_interval": [
              266,
              274
            ]
          },
          "prefix_rank": {
            "rank": 277,
            "rank_pessimistic": 277,
            "near_tie_rank_interval": [
              272,
              281
            ]
          },
          "numerical_check": {
            "cached_single": [
              -16.14556884765625,
              -16.14550018310547,
              -6.735097849741578e-05
            ],
            "uncached_single": [
              -16.3612003326416,
              -16.361122131347656,
              -7.939023635117337e-05
            ],
            "cached_batch": [
              -16.23769760131836,
              -16.237628936767578,
              -6.770858453819528e-05
            ],
            "cache_vs_full_max_error": 0.21563148498535156
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 157,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "taccio finché posso",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto in voce",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto al telefono",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "taccio finché non posso",
              "correct": false
            }
          ],
          "logprob": -10.55595874786377,
          "logprob_with_eos": -10.557768821716309,
          "rank": {
            "rank": 11,
            "rank_pessimistic": 11,
            "near_tie_rank_interval": [
              11,
              11
            ]
          },
          "prefix_rank": {
            "rank": 11,
            "rank_pessimistic": 11,
            "near_tie_rank_interval": [
              11,
              11
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.56452465057373,
              -10.562715530395508,
              -0.0018091516103595495
            ],
            "uncached_single": [
              -10.541844367980957,
              -10.540030479431152,
              -0.0018135543214157224
            ],
            "cached_batch": [
              -10.557768821716309,
              -10.55595874786377,
              -0.0018099845619872212
            ],
            "cache_vs_full_max_error": 0.02268505096435547
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 172,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta figura in testa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "l'ora della Femmina",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brutta figura in mano",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fata malinconsigliata",
              "correct": false
            }
          ],
          "logprob": -11.99332046508789,
          "logprob_with_eos": -13.811887741088867,
          "rank": {
            "rank": 30,
            "rank_pessimistic": 30,
            "near_tie_rank_interval": [
              29,
              30
            ]
          },
          "prefix_rank": {
            "rank": 38,
            "rank_pessimistic": 38,
            "near_tie_rank_interval": [
              37,
              38
            ]
          },
          "numerical_check": {
            "cached_single": [
              -13.634939193725586,
              -11.92646598815918,
              -1.7084730863571167
            ],
            "uncached_single": [
              -13.768999099731445,
              -12.002052307128906,
              -1.766947865486145
            ],
            "cached_batch": [
              -13.811887741088867,
              -11.99332046508789,
              -1.8185675144195557
            ],
            "cache_vs_full_max_error": 0.13405990600585938
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 15,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 30,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.4,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": true,
      "adapter_seed": 17,
      "adapter_sha256": {
        "README.md": "b24b74667d1eb9c991047b4507c329613ae38707f4450446a94217081af69e96",
        "adapter_config.json": "c30fd850e6a43cb509536a3ad2b6e4939115e18dab7e71f5e7161ebac9037ec2",
        "adapter_model.safetensors": "20577d4479d454180dbd1571eda4e18fa2e2942b27960a9703a4d3b15dc73657",
        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
        "tokenizer.json": "a5cd9732badce41de57e6efce8302930ded1c1188c5f81feb2bd6c24c4a1941f",
        "tokenizer_config.json": "f6fdf337c2c5f4ed8190168a772c83f5c9364d225abae63223fde372e3cfb71f"
      },
      "prompt_source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 262.786729995,
      "gpu": "NVIDIA A100-SXM4-80GB",
      "source_sha256": "40036973521582e377746b371583eb3e66ab6dbe8671ea5e03179fe37c54d447",
      "plan_sha256": "ae270c72f21785f2396a35f09d83cc62b1f7f495d5fe690ef633083e96112239",
      "protocol_sha256": "12d5d8c26e2557fd1e4d255ae478de508b7eaaea3462b3a4f2673086d9fd3b98",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen35-27b-s17": {
      "complete": true,
      "model": "Qwen/Qwen3.5-27B",
      "revision": "fc05daec18b0a78c049392ed2e771dde82bdf654",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 44,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "comparsa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "scoperte",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "scopertine",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "comparsa",
              "correct": false
            }
          ],
          "logprob": -15.397603988647461,
          "logprob_with_eos": -15.397660255432129,
          "rank": {
            "rank": 57,
            "rank_pessimistic": 57,
            "near_tie_rank_interval": [
              57,
              61
            ]
          },
          "prefix_rank": {
            "rank": 57,
            "rank_pessimistic": 57,
            "near_tie_rank_interval": [
              57,
              61
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.319711685180664,
              -15.319656372070312,
              -5.61460001335945e-05
            ],
            "uncached_single": [
              -15.340240478515625,
              -15.34018325805664,
              -5.674201020156033e-05
            ],
            "cached_batch": [
              -15.397660255432129,
              -15.397603988647461,
              -5.686121585313231e-05
            ],
            "cache_vs_full_max_error": 0.020528793334960938
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 42,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "slow transmission",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "slow decomposition",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "slow disintegration",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "very spontaneous",
              "correct": false
            }
          ],
          "logprob": -8.427528381347656,
          "logprob_with_eos": -8.428083419799805,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -8.461803436279297,
              -8.461174011230469,
              -0.0006293461774475873
            ],
            "uncached_single": [
              -8.49130630493164,
              -8.490676879882812,
              -0.0006298227235674858
            ],
            "cached_batch": [
              -8.428083419799805,
              -8.427528381347656,
              -0.0005547653418034315
            ],
            "cache_vs_full_max_error": 0.02950286865234375
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 47,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "fondamenta capannone",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "costrutti grammaticali",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "fondamenta lagunari",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenti costruiti",
              "correct": false
            }
          ],
          "logprob": -12.441546440124512,
          "logprob_with_eos": -12.441620826721191,
          "rank": {
            "rank": 18,
            "rank_pessimistic": 18,
            "near_tie_rank_interval": [
              18,
              19
            ]
          },
          "prefix_rank": {
            "rank": 19,
            "rank_pessimistic": 19,
            "near_tie_rank_interval": [
              19,
              20
            ]
          },
          "numerical_check": {
            "cached_single": [
              -12.51255989074707,
              -12.51248550415039,
              -7.331102824537084e-05
            ],
            "uncached_single": [
              -12.540630340576172,
              -12.540557861328125,
              -7.295342220459133e-05
            ],
            "cached_batch": [
              -12.441620826721191,
              -12.441546440124512,
              -7.414542778860778e-05
            ],
            "cache_vs_full_max_error": 0.028072357177734375
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 49,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "muto affare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto tale",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto nero",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "tengo bocca",
              "correct": false
            }
          ],
          "logprob": -9.506025314331055,
          "logprob_with_eos": -9.506278038024902,
          "rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "prefix_rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "numerical_check": {
            "cached_single": [
              -9.457432746887207,
              -9.457182884216309,
              -0.00024971229140646756
            ],
            "uncached_single": [
              -9.523642539978027,
              -9.523392677307129,
              -0.00024971229140646756
            ],
            "cached_batch": [
              -9.506278038024902,
              -9.506025314331055,
              -0.0002530493075028062
            ],
            "cache_vs_full_max_error": 0.06620979309082031
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 64,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta manipola",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una brutta manipola",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brutta manovra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta manipola",
              "correct": false
            }
          ],
          "logprob": -4.93061637878418,
          "logprob_with_eos": -4.935867786407471,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -4.933314323425293,
              -4.927435874938965,
              -0.0058784219436347485
            ],
            "uncached_single": [
              -4.938960075378418,
              -4.934271335601807,
              -0.00468869786709547
            ],
            "cached_batch": [
              -4.935867786407471,
              -4.93061637878418,
              -0.005251778755337
            ],
            "cache_vs_full_max_error": 0.006835460662841797
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 152,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "comparsa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "scoperte",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "scoperte nuove",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "complesso",
              "correct": false
            }
          ],
          "logprob": -14.663423538208008,
          "logprob_with_eos": -14.663618087768555,
          "rank": {
            "rank": 48,
            "rank_pessimistic": 48,
            "near_tie_rank_interval": [
              48,
              49
            ]
          },
          "prefix_rank": {
            "rank": 49,
            "rank_pessimistic": 49,
            "near_tie_rank_interval": [
              49,
              50
            ]
          },
          "numerical_check": {
            "cached_single": [
              -14.581893920898438,
              -14.581705093383789,
              -0.00019000156316906214
            ],
            "uncached_single": [
              -14.580568313598633,
              -14.580374717712402,
              -0.00019369633810129017
            ],
            "cached_batch": [
              -14.663618087768555,
              -14.663423538208008,
              -0.00019405389321036637
            ],
            "cache_vs_full_max_error": 0.0013303756713867188
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 150,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "slow radiation",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "slow decomposition",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "slow disintegration",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "slow transformation",
              "correct": false
            }
          ],
          "logprob": -8.388885498046875,
          "logprob_with_eos": -8.389484405517578,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -8.455810546875,
              -8.45522403717041,
              -0.000586695212405175
            ],
            "uncached_single": [
              -8.445672988891602,
              -8.445108413696289,
              -0.000565249880310148
            ],
            "cached_batch": [
              -8.389484405517578,
              -8.388885498046875,
              -0.000598251644987613
            ],
            "cache_vs_full_max_error": 0.010137557983398438
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 155,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "fondamenta sospese",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "costrutti grammaticali",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "fondamenta lagunari",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenti costruiti",
              "correct": false
            }
          ],
          "logprob": -13.027427673339844,
          "logprob_with_eos": -13.027524948120117,
          "rank": {
            "rank": 21,
            "rank_pessimistic": 21,
            "near_tie_rank_interval": [
              20,
              22
            ]
          },
          "prefix_rank": {
            "rank": 23,
            "rank_pessimistic": 23,
            "near_tie_rank_interval": [
              22,
              23
            ]
          },
          "numerical_check": {
            "cached_single": [
              -13.041513442993164,
              -13.04141902923584,
              -9.440929716220126e-05
            ],
            "uncached_single": [
              -13.011151313781738,
              -13.011051177978516,
              -0.00010024998482549563
            ],
            "cached_batch": [
              -13.027524948120117,
              -13.027427673339844,
              -9.691245941212401e-05
            ],
            "cache_vs_full_max_error": 0.03036785125732422
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 157,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "muto, però!",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto tale",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto vero",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "tengo muta",
              "correct": false
            }
          ],
          "logprob": -10.135994911193848,
          "logprob_with_eos": -10.136420249938965,
          "rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "prefix_rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.034332275390625,
              -10.033903121948242,
              -0.0004287039046175778
            ],
            "uncached_single": [
              -10.01002025604248,
              -10.009597778320312,
              -0.00042226933874189854
            ],
            "cached_batch": [
              -10.136420249938965,
              -10.135994911193848,
              -0.00042524831951595843
            ],
            "cache_vs_full_max_error": 0.02431201934814453
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 172,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta impressione",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una brutta manipola",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brutta manovella",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta manipola",
              "correct": false
            }
          ],
          "logprob": -4.805523872375488,
          "logprob_with_eos": -4.81049919128418,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -4.768953323364258,
              -4.76334810256958,
              -0.0056052180007100105
            ],
            "uncached_single": [
              -4.777921676635742,
              -4.772308349609375,
              -0.005613041575998068
            ],
            "cached_batch": [
              -4.81049919128418,
              -4.805523872375488,
              -0.004974963143467903
            ],
            "cache_vs_full_max_error": 0.008968353271484375
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 4,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.6,
            "10": 0.6,
            "20": 0.8,
            "50": 0.8,
            "100": 1.0,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 4,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.6,
            "10": 0.6,
            "20": 0.6,
            "50": 1.0,
            "100": 1.0,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": true,
      "adapter_seed": 17,
      "adapter_sha256": {
        "README.md": "32aebccad2ad18048f1963e75c16dbba0f78e5ab57410415ff92a30e20574bc8",
        "adapter_config.json": "0934ed5b6687018508cdb5269855d1529db2e1c3ffc008553dae627a4b4c2737",
        "adapter_model.safetensors": "b2879f81eb53277c96c4520a78fe28efadddd4a35c40262666c7f19d2ec097de",
        "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715",
        "tokenizer.json": "a5cd9732badce41de57e6efce8302930ded1c1188c5f81feb2bd6c24c4a1941f",
        "tokenizer_config.json": "f6fdf337c2c5f4ed8190168a772c83f5c9364d225abae63223fde372e3cfb71f"
      },
      "prompt_source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 299.425763223,
      "gpu": "NVIDIA H100 80GB HBM3",
      "source_sha256": "40036973521582e377746b371583eb3e66ab6dbe8671ea5e03179fe37c54d447",
      "plan_sha256": "ae270c72f21785f2396a35f09d83cc62b1f7f495d5fe690ef633083e96112239",
      "protocol_sha256": "12d5d8c26e2557fd1e4d255ae478de508b7eaaea3462b3a4f2673086d9fd3b98",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen38-27b-s17": {
      "complete": true,
      "model": "Qwen/Qwen3.8-27B",
      "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 44,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "collettivi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "imprevisti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "collettivi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "complesso",
              "correct": false
            }
          ],
          "logprob": -19.242019653320312,
          "logprob_with_eos": -19.242034912109375,
          "rank": {
            "rank": 84,
            "rank_pessimistic": 84,
            "near_tie_rank_interval": [
              81,
              84
            ]
          },
          "prefix_rank": {
            "rank": 84,
            "rank_pessimistic": 84,
            "near_tie_rank_interval": [
              81,
              84
            ]
          },
          "numerical_check": {
            "cached_single": [
              -19.144451141357422,
              -19.144437789916992,
              -1.4781842764932662e-05
            ],
            "uncached_single": [
              -19.225238800048828,
              -19.225223541259766,
              -1.4424220353248529e-05
            ],
            "cached_batch": [
              -19.242034912109375,
              -19.242019653320312,
              -1.4543427823809907e-05
            ],
            "cache_vs_full_max_error": 0.08078765869140625
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 42,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Uran, decomposed",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Uran's succession",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "slow disintegration",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "long reduction",
              "correct": false
            }
          ],
          "logprob": -7.354621887207031,
          "logprob_with_eos": -7.355931282043457,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -7.355725288391113,
              -7.3544158935546875,
              -0.0013093716697767377
            ],
            "uncached_single": [
              -7.439787864685059,
              -7.438479423522949,
              -0.0013090145075693727
            ],
            "cached_batch": [
              -7.355931282043457,
              -7.354621887207031,
              -0.0013092526933178306
            ],
            "cache_vs_full_max_error": 0.08406352996826172
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 47,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "fondamentali insediamenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "colonnine portanti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "colonnine sostenute",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenti costruttivi",
              "correct": false
            }
          ],
          "logprob": -14.348123550415039,
          "logprob_with_eos": -14.348254203796387,
          "rank": {
            "rank": 47,
            "rank_pessimistic": 47,
            "near_tie_rank_interval": [
              46,
              47
            ]
          },
          "prefix_rank": {
            "rank": 48,
            "rank_pessimistic": 48,
            "near_tie_rank_interval": [
              47,
              48
            ]
          },
          "numerical_check": {
            "cached_single": [
              -14.325357437133789,
              -14.325224876403809,
              -0.00013219437096267939
            ],
            "uncached_single": [
              -14.392906188964844,
              -14.39278793334961,
              -0.00011848701251437888
            ],
            "cached_batch": [
              -14.348254203796387,
              -14.348123550415039,
              -0.00013040646445006132
            ],
            "cache_vs_full_max_error": 0.06756305694580078
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 49,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "muto, caro!",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto fine",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto, eh?",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "tengo mute",
              "correct": false
            }
          ],
          "logprob": -8.92241382598877,
          "logprob_with_eos": -8.923273086547852,
          "rank": {
            "rank": 3,
            "rank_pessimistic": 3,
            "near_tie_rank_interval": [
              3,
              3
            ]
          },
          "prefix_rank": {
            "rank": 3,
            "rank_pessimistic": 3,
            "near_tie_rank_interval": [
              3,
              3
            ]
          },
          "numerical_check": {
            "cached_single": [
              -8.909875869750977,
              -8.909014701843262,
              -0.0008609164506196976
            ],
            "uncached_single": [
              -8.951611518859863,
              -8.950752258300781,
              -0.0008592489175498486
            ],
            "cached_batch": [
              -8.923273086547852,
              -8.92241382598877,
              -0.0008593680104240775
            ],
            "cache_vs_full_max_error": 0.04173755645751953
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 64,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta piega",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una brutta piega",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brutta passata",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta piega",
              "correct": false
            }
          ],
          "logprob": -5.478691101074219,
          "logprob_with_eos": -5.48567533493042,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -5.530520439147949,
              -5.524075508117676,
              -0.006444860715419054
            ],
            "uncached_single": [
              -5.545519828796387,
              -5.538532257080078,
              -0.006987776607275009
            ],
            "cached_batch": [
              -5.48567533493042,
              -5.478691101074219,
              -0.006984580308198929
            ],
            "cache_vs_full_max_error": 0.0149993896484375
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 152,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "complesse",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "imprevisti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "collettiva",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "complesse",
              "correct": false
            }
          ],
          "logprob": -20.430633544921875,
          "logprob_with_eos": -20.430675506591797,
          "rank": {
            "rank": 99,
            "rank_pessimistic": 99,
            "near_tie_rank_interval": [
              95,
              100
            ]
          },
          "prefix_rank": {
            "rank": 99,
            "rank_pessimistic": 99,
            "near_tie_rank_interval": [
              96,
              100
            ]
          },
          "numerical_check": {
            "cached_single": [
              -20.376089096069336,
              -20.376049041748047,
              -4.029192859889008e-05
            ],
            "uncached_single": [
              -20.423803329467773,
              -20.423763275146484,
              -3.957670196541585e-05
            ],
            "cached_batch": [
              -20.430675506591797,
              -20.430633544921875,
              -4.172238186583854e-05
            ],
            "cache_vs_full_max_error": 0.0477142333984375
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 150,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "slow disintegration",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "fall transition",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "slow disintegration",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "uran, transformation",
              "correct": false
            }
          ],
          "logprob": -6.488040447235107,
          "logprob_with_eos": -6.4891743659973145,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -6.479389667510986,
              -6.478255748748779,
              -0.001134109916165471
            ],
            "uncached_single": [
              -6.494790077209473,
              -6.493655204772949,
              -0.001134705264121294
            ],
            "cached_batch": [
              -6.4891743659973145,
              -6.488040447235107,
              -0.0011338717304170132
            ],
            "cache_vs_full_max_error": 0.015400409698486328
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 155,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "fondamenti costruttivi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "colonnine portanti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "colonnine affondate",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondente montato",
              "correct": false
            }
          ],
          "logprob": -14.442325592041016,
          "logprob_with_eos": -14.44240951538086,
          "rank": {
            "rank": 49,
            "rank_pessimistic": 49,
            "near_tie_rank_interval": [
              48,
              51
            ]
          },
          "prefix_rank": {
            "rank": 50,
            "rank_pessimistic": 50,
            "near_tie_rank_interval": [
              49,
              52
            ]
          },
          "numerical_check": {
            "cached_single": [
              -14.491082191467285,
              -14.490996360778809,
              -8.582700684200972e-05
            ],
            "uncached_single": [
              -14.477842330932617,
              -14.477751731872559,
              -9.035655966727063e-05
            ],
            "cached_batch": [
              -14.44240951538086,
              -14.442325592041016,
              -8.415821503149346e-05
            ],
            "cache_vs_full_max_error": 0.01324462890625
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 157,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "muto, eh?",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto fine",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto, eh?",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "freno mano",
              "correct": false
            }
          ],
          "logprob": -9.701809883117676,
          "logprob_with_eos": -9.704829216003418,
          "rank": {
            "rank": 3,
            "rank_pessimistic": 3,
            "near_tie_rank_interval": [
              3,
              3
            ]
          },
          "prefix_rank": {
            "rank": 3,
            "rank_pessimistic": 3,
            "near_tie_rank_interval": [
              3,
              3
            ]
          },
          "numerical_check": {
            "cached_single": [
              -9.66654109954834,
              -9.663525581359863,
              -0.0030157347209751606
            ],
            "uncached_single": [
              -9.66189956665039,
              -9.658489227294922,
              -0.003409408265724778
            ],
            "cached_batch": [
              -9.704829216003418,
              -9.701809883117676,
              -0.0030189435929059982
            ],
            "cache_vs_full_max_error": 0.005036354064941406
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 172,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una brutta piega pubblica",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una brutta piega pubblica",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una brutta passata",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una brutta manovra",
              "correct": false
            }
          ],
          "logprob": -5.077340126037598,
          "logprob_with_eos": -5.08863639831543,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -5.122507095336914,
              -5.109745979309082,
              -0.012761005200445652
            ],
            "uncached_single": [
              -5.114790916442871,
              -5.103498458862305,
              -0.011292001232504845
            ],
            "cached_batch": [
              -5.08863639831543,
              -5.077340126037598,
              -0.011296833865344524
            ],
            "cache_vs_full_max_error": 0.007716178894042969
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 3,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.6,
            "5": 0.6,
            "10": 0.6,
            "20": 0.6,
            "50": 0.8,
            "100": 1.0,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 3,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.6,
            "5": 0.6,
            "10": 0.6,
            "20": 0.6,
            "50": 0.8,
            "100": 1.0,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": true,
      "adapter_seed": 17,
      "adapter_sha256": {
        "README.md": "62bc65e50d6d930922648b14bb542393430dc4f5300a3eb0f63e6e97fc659077",
        "adapter_config.json": "fdbd2ca46e5fe91e90d2cd2c1f4716ccfb2cbfde05ce3c1dfe49d6f25aab1618",
        "adapter_model.safetensors": "488a19865ab2d42c73b6d7033753308ace1f1f3b512605a3e020986bc787140f",
        "chat_template.jinja": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
        "tokenizer.json": "a5cd9732badce41de57e6efce8302930ded1c1188c5f81feb2bd6c24c4a1941f",
        "tokenizer_config.json": "f6fdf337c2c5f4ed8190168a772c83f5c9364d225abae63223fde372e3cfb71f"
      },
      "prompt_source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 369.021928119,
      "gpu": "NVIDIA H100 80GB HBM3",
      "source_sha256": "40036973521582e377746b371583eb3e66ab6dbe8671ea5e03179fe37c54d447",
      "plan_sha256": "ae270c72f21785f2396a35f09d83cc62b1f7f495d5fe690ef633083e96112239",
      "protocol_sha256": "12d5d8c26e2557fd1e4d255ae478de508b7eaaea3462b3a4f2673086d9fd3b98",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    }
  },
  "definition": "Una crittografia mnemonica è un gioco enigmistico basato sul doppio senso. Si interpreta il significato dell’esposto per trovare una parola o espressione che lo descriva e che abbia anche un’altra lettura autonoma e riconoscibile. Il passaggio è semantico: non richiede di cifrare le lettere, assegnare loro numeri o riordinarle in un anagramma. Il diagramma numerico indica le lunghezze delle parole della soluzione. La risposta deve rispettare il diagramma e sostenere entrambe le letture.",
  "candidate_count": 2008,
  "cases": [
    {
      "id": "palafitte",
      "answer": "palafitte",
      "prompt": "Risolvi questa crittografia mnemonica italiana: proprietà emergenti (9).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    },
    {
      "id": "uranium",
      "answer": "lead generation",
      "prompt": "Solve this English mnemonic cryptogram: Natural Uranium Decay (4, 10).\nWrite only the solution, without preamble or explanation."
    },
    {
      "id": "inverse",
      "answer": "proprietà emergenti",
      "prompt": "Risolvi questa crittografia mnemonica italiana: PALAFITTE (9 9).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    },
    {
      "id": "speech",
      "answer": "servo muto",
      "prompt": "Risolvi questa crittografia mnemonica italiana: NON POSSO PARLARE (5 4).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    },
    {
      "id": "influenza",
      "answer": "una brutta influenza",
      "prompt": "Risolvi questa crittografia mnemonica italiana: UNA DONNA ORRIBILE MANIPOLA L'OPINIONE PUBBLICA (3; 6; 9).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    }
  ]
}