{
  "context": {
    "qwen3-4b": {
      "complete": true,
      "model": "Qwen/Qwen3-4B-Instruct-2507",
      "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 43,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Proprietà emergenti: il tutto è diverso dalla somma delle parti.",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Fisica",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Fisica, chimica, biologia, matematica, informatica, psicologia, economia, sociologia, storia.",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Proprietà emergenti: il sistema si comporta in modo non previsto, ma coerente.",
              "correct": false
            }
          ],
          "logprob": -27.988666534423828,
          "logprob_with_eos": -27.98881721496582,
          "rank": {
            "rank": 147,
            "rank_pessimistic": 147,
            "near_tie_rank_interval": [
              143,
              148
            ]
          },
          "prefix_rank": {
            "rank": 154,
            "rank_pessimistic": 154,
            "near_tie_rank_interval": [
              150,
              157
            ]
          },
          "numerical_check": {
            "cached_single": [
              -28.161865234375,
              -28.161697387695312,
              -0.0001668790791882202
            ],
            "uncached_single": [
              -27.577333450317383,
              -27.57719612121582,
              -0.00013779645087197423
            ],
            "cached_batch": [
              -27.98881721496582,
              -27.988666534423828,
              -0.0001512651506345719
            ],
            "cache_vs_full_max_error": 0.5845317840576172
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 37,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Alpha, Beta",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Uranium-238, Thorium-232",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Actinium, Thorium",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Alpha, beta",
              "correct": false
            }
          ],
          "logprob": -37.673500061035156,
          "logprob_with_eos": -37.67350387573242,
          "rank": {
            "rank": 14,
            "rank_pessimistic": 14,
            "near_tie_rank_interval": [
              14,
              14
            ]
          },
          "prefix_rank": {
            "rank": 14,
            "rank_pessimistic": 14,
            "near_tie_rank_interval": [
              14,
              14
            ]
          },
          "numerical_check": {
            "cached_single": [
              -37.275508880615234,
              -37.27550506591797,
              -3.4570634852570947e-06
            ],
            "uncached_single": [
              -36.94229507446289,
              -36.942291259765625,
              -4.410734163684538e-06
            ],
            "cached_batch": [
              -37.67350387573242,
              -37.673500061035156,
              -3.4570634852570947e-06
            ],
            "cache_vs_full_max_error": 0.33321380615234375
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 46,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "PALAFITTE → P A L A F I T T E  \nNumeri: (9 9) → 9 lettere, 9 lettere  \n\nAnalisi mnemonica: ogni lettera corrisponde a un",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FILATI PALLA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "PALAFITTE → P-A-L-A-F-I-T-T-E  \n(9 9) → 9 lettere, 9 lettere  \n\nAnalisi: ogni lettera corrisponde a un numero (A=1,",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "PALAFITTE → P-A-L-A-F-I-T-T-E  \n(9 9) → 9 lettere, 9 lettere  \n\nOsserviamo che la parola ha 9 lettere e il numero ",
              "correct": false
            }
          ],
          "logprob": -29.90350341796875,
          "logprob_with_eos": -29.905609130859375,
          "rank": {
            "rank": 179,
            "rank_pessimistic": 179,
            "near_tie_rank_interval": [
              178,
              179
            ]
          },
          "prefix_rank": {
            "rank": 184,
            "rank_pessimistic": 184,
            "near_tie_rank_interval": [
              183,
              184
            ]
          },
          "numerical_check": {
            "cached_single": [
              -29.762393951416016,
              -29.76031494140625,
              -0.0020754244178533554
            ],
            "uncached_single": [
              -29.871706008911133,
              -29.869321823120117,
              -0.0023834886960685253
            ],
            "cached_batch": [
              -29.905609130859375,
              -29.90350341796875,
              -0.0021069487556815147
            ],
            "cache_vs_full_max_error": 0.10931205749511719
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 48,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "CERCO PALLA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "CERCO PIAZZA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Vedi il testo e conta le lettere: NON POSSO PARLARE → 15 lettere. Ma il suggerimento è (5 4), quindi prendi la 5ª e la 4ª letter",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Vedi il numero 5 e 4 come posizioni nelle parole.  \nNON (1 2 3 4) → posizione 5: non esiste → saltare  \nPOSSO (5 4 3",
              "correct": false
            }
          ],
          "logprob": -29.210277557373047,
          "logprob_with_eos": -29.214027404785156,
          "rank": {
            "rank": 130,
            "rank_pessimistic": 130,
            "near_tie_rank_interval": [
              126,
              132
            ]
          },
          "prefix_rank": {
            "rank": 135,
            "rank_pessimistic": 135,
            "near_tie_rank_interval": [
              132,
              138
            ]
          },
          "numerical_check": {
            "cached_single": [
              -29.077880859375,
              -29.07410430908203,
              -0.0037791754584759474
            ],
            "uncached_single": [
              -29.253643035888672,
              -29.250003814697266,
              -0.003637841437011957
            ],
            "cached_batch": [
              -29.214027404785156,
              -29.210277557373047,
              -0.0037472285330295563
            ],
            "cache_vs_full_max_error": 0.17589950561523438
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 66,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "CIAO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "CIAO Mondo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "CIAO MENTE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "CIAO Mondo",
              "correct": false
            }
          ],
          "logprob": -28.961761474609375,
          "logprob_with_eos": -28.964679718017578,
          "rank": {
            "rank": 264,
            "rank_pessimistic": 264,
            "near_tie_rank_interval": [
              258,
              267
            ]
          },
          "prefix_rank": {
            "rank": 288,
            "rank_pessimistic": 288,
            "near_tie_rank_interval": [
              284,
              290
            ]
          },
          "numerical_check": {
            "cached_single": [
              -28.982393264770508,
              -28.979951858520508,
              -0.002441046992316842
            ],
            "uncached_single": [
              -28.99374008178711,
              -28.991294860839844,
              -0.0024441389832645655
            ],
            "cached_batch": [
              -28.964679718017578,
              -28.961761474609375,
              -0.0029214818496257067
            ],
            "cache_vs_full_max_error": 0.011346817016601562
          }
        },
        {
          "condition": "examples",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 1911,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "elementi in movimento",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "caratteristiche visibili",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "fenomeni naturali",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "elementi che si fanno noti",
              "correct": false
            }
          ],
          "logprob": -19.00556182861328,
          "logprob_with_eos": -34.589595794677734,
          "rank": {
            "rank": 1229,
            "rank_pessimistic": 1229,
            "near_tie_rank_interval": [
              1226,
              1230
            ]
          },
          "prefix_rank": {
            "rank": 114,
            "rank_pessimistic": 114,
            "near_tie_rank_interval": [
              114,
              114
            ]
          },
          "numerical_check": {
            "cached_single": [
              -34.50060272216797,
              -19.102680206298828,
              -15.397921562194824
            ],
            "uncached_single": [
              -34.617401123046875,
              -19.09295082092285,
              -15.524447441101074
            ],
            "cached_batch": [
              -34.589595794677734,
              -19.00556182861328,
              -15.584033012390137
            ],
            "cache_vs_full_max_error": 0.12652587890625
          }
        },
        {
          "condition": "examples",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 1905,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "alpha decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "alpha decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "alpha decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "alpha decay",
              "correct": false
            }
          ],
          "logprob": -21.2896785736084,
          "logprob_with_eos": -21.291748046875,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -21.476051330566406,
              -21.47372817993164,
              -0.00232081301510334
            ],
            "uncached_single": [
              -21.24944496154785,
              -21.24738311767578,
              -0.002063171239569783
            ],
            "cached_batch": [
              -21.291748046875,
              -21.2896785736084,
              -0.0020690003875643015
            ],
            "cache_vs_full_max_error": 0.2266063690185547
          }
        },
        {
          "condition": "examples",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 1914,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "pali in posizione fissa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "pali in posizione fissa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "piattaforma di sostegno",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "piattaforma di supporto",
              "correct": false
            }
          ],
          "logprob": -23.150489807128906,
          "logprob_with_eos": -23.217222213745117,
          "rank": {
            "rank": 375,
            "rank_pessimistic": 375,
            "near_tie_rank_interval": [
              369,
              378
            ]
          },
          "prefix_rank": {
            "rank": 414,
            "rank_pessimistic": 414,
            "near_tie_rank_interval": [
              407,
              421
            ]
          },
          "numerical_check": {
            "cached_single": [
              -23.519227981567383,
              -23.45260238647461,
              -0.06662460416555405
            ],
            "uncached_single": [
              -23.328754425048828,
              -23.26241683959961,
              -0.06633961945772171
            ],
            "cached_batch": [
              -23.217222213745117,
              -23.150489807128906,
              -0.06673289090394974
            ],
            "cache_vs_full_max_error": 0.1904735565185547
          }
        },
        {
          "condition": "examples",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 1916,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "non ho voce",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "non ho voce",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "non posso più parlare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "non posso più parlare",
              "correct": false
            }
          ],
          "logprob": -25.629867553710938,
          "logprob_with_eos": -25.67323112487793,
          "rank": {
            "rank": 441,
            "rank_pessimistic": 441,
            "near_tie_rank_interval": [
              433,
              442
            ]
          },
          "prefix_rank": {
            "rank": 470,
            "rank_pessimistic": 470,
            "near_tie_rank_interval": [
              463,
              477
            ]
          },
          "numerical_check": {
            "cached_single": [
              -25.651880264282227,
              -25.609912872314453,
              -0.04196765273809433
            ],
            "uncached_single": [
              -25.84168815612793,
              -25.795286178588867,
              -0.04640217497944832
            ],
            "cached_batch": [
              -25.67323112487793,
              -25.629867553710938,
              -0.04336228966712952
            ],
            "cache_vs_full_max_error": 0.18980789184570312
          }
        },
        {
          "condition": "examples",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 1934,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "donna con paura del pubblico",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una donna che parla",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "c'è un'ora di tregua",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fatica di governo",
              "correct": false
            }
          ],
          "logprob": -16.518470764160156,
          "logprob_with_eos": -16.518646240234375,
          "rank": {
            "rank": 39,
            "rank_pessimistic": 39,
            "near_tie_rank_interval": [
              39,
              39
            ]
          },
          "prefix_rank": {
            "rank": 49,
            "rank_pessimistic": 49,
            "near_tie_rank_interval": [
              48,
              50
            ]
          },
          "numerical_check": {
            "cached_single": [
              -16.54610252380371,
              -16.54596710205078,
              -0.00013696208770852536
            ],
            "uncached_single": [
              -16.504867553710938,
              -16.504741668701172,
              -0.00012516192509792745
            ],
            "cached_batch": [
              -16.518646240234375,
              -16.518470764160156,
              -0.0001746263587847352
            ],
            "cache_vs_full_max_error": 0.04123497009277344
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 187,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "proprietà emergenti  \n(9) → 9 lettere\n\nLetture:\n- Proprietà emergenti → 9 lettere\n- \"Proprietà emergenti\" ha 9 lettere: P-R-O-P",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "proprietà emergenti  \n(9) → 9 lettere\n\nLetture:\n- Proprietà emergenti → \"proprietà emergenti\" (9 lettere? No, ha più di 9)\nRiconsider",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "proprietà emergenti  \n(9) → 9 lettere\n\nLetture:\n- \"proprietà emergenti\" → parola che descrive il concetto (proprietà emergenti)\n- Lettura auton",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "proprietà emergenti  \nproprietà emergenti",
              "correct": false
            }
          ],
          "logprob": -25.411808013916016,
          "logprob_with_eos": -35.86466979980469,
          "rank": {
            "rank": 729,
            "rank_pessimistic": 729,
            "near_tie_rank_interval": [
              724,
              732
            ]
          },
          "prefix_rank": {
            "rank": 89,
            "rank_pessimistic": 89,
            "near_tie_rank_interval": [
              88,
              91
            ]
          },
          "numerical_check": {
            "cached_single": [
              -35.29690170288086,
              -24.930789947509766,
              -10.366113662719727
            ],
            "uncached_single": [
              -35.095367431640625,
              -24.932384490966797,
              -10.162980079650879
            ],
            "cached_batch": [
              -35.86466979980469,
              -25.411808013916016,
              -10.452865600585938
            ],
            "cache_vs_full_max_error": 0.20313358306884766
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 181,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "lead decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "lead decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "lead decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "lead decay",
              "correct": false
            }
          ],
          "logprob": -17.57924461364746,
          "logprob_with_eos": -17.57924461364746,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -17.559751510620117,
              -17.559749603271484,
              -1.311301275563892e-06
            ],
            "uncached_single": [
              -17.598094940185547,
              -17.598094940185547,
              -1.1920922133867862e-06
            ],
            "cached_batch": [
              -17.57924461364746,
              -17.57924461364746,
              -1.311301275563892e-06
            ],
            "cache_vs_full_max_error": 0.0383453369140625
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 190,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "PALAZZO FESTA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "PALAZZO FESTA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "PALAZZO FISSE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "PALAFITTE  \nPALAFITTE",
              "correct": false
            }
          ],
          "logprob": -32.381103515625,
          "logprob_with_eos": -32.381103515625,
          "rank": {
            "rank": 439,
            "rank_pessimistic": 439,
            "near_tie_rank_interval": [
              436,
              443
            ]
          },
          "prefix_rank": {
            "rank": 448,
            "rank_pessimistic": 448,
            "near_tie_rank_interval": [
              445,
              452
            ]
          },
          "numerical_check": {
            "cached_single": [
              -32.12923049926758,
              -32.12923049926758,
              -1.0728830375228426e-06
            ],
            "uncached_single": [
              -32.170162200927734,
              -32.17015838623047,
              -1.0728830375228426e-06
            ],
            "cached_batch": [
              -32.381103515625,
              -32.381103515625,
              -1.0728830375228426e-06
            ],
            "cache_vs_full_max_error": 0.04093170166015625
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 192,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "non posso parlare  \nnò posso parlàre",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "non pò parlàre",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Nona poca parlare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "non posso parlare  \nnò posso parlàre",
              "correct": false
            }
          ],
          "logprob": -27.860206604003906,
          "logprob_with_eos": -27.860210418701172,
          "rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              25,
              26
            ]
          },
          "prefix_rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              25,
              26
            ]
          },
          "numerical_check": {
            "cached_single": [
              -28.125850677490234,
              -28.1258487701416,
              -2.861018856492592e-06
            ],
            "uncached_single": [
              -27.631399154663086,
              -27.631397247314453,
              -2.3841830625315197e-06
            ],
            "cached_batch": [
              -27.860210418701172,
              -27.860206604003906,
              -2.9802276912960224e-06
            ],
            "cache_vs_full_max_error": 0.49445152282714844
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 210,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una donna orribile manipola l'opinione pubblica  \n→ una / donna / orribile  \n→ una / donna / orribile  \n\nMa la lettura alternativa deve essere riconoscibile e auton",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "una donna orribile manipola l'opinione pubblica  \n→ una / donna / orribile  \n→ una / donna / orribile  \n\nMa la lettura alternativa deve essere riconoscibile e auton",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una donna orribile manipola l'opinione pubblica  \n→ una / donna / orribile  \n→ una / donna / orribile  \n\nMa la lettura alternativa:  \n\"una donna orribile\"",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "una donna orribile manipola l'opinione pubblica  \n→ una / donna / orribile  \n→ **una** / **donna** / **orribile**  \n\nMa dobbiamo trovare una parola",
              "correct": false
            }
          ],
          "logprob": -19.26629066467285,
          "logprob_with_eos": -19.266429901123047,
          "rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "prefix_rank": {
            "rank": 7,
            "rank_pessimistic": 7,
            "near_tie_rank_interval": [
              7,
              7
            ]
          },
          "numerical_check": {
            "cached_single": [
              -19.43941879272461,
              -19.439266204833984,
              -0.0001517419150331989
            ],
            "uncached_single": [
              -19.45813751220703,
              -19.457969665527344,
              -0.0001668790791882202
            ],
            "cached_batch": [
              -19.266429901123047,
              -19.26629066467285,
              -0.0001387499796692282
            ],
            "cache_vs_full_max_error": 0.018718719482421875
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 147,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.0,
            "5": 0.0,
            "10": 0.0,
            "20": 0.2,
            "50": 0.2,
            "100": 0.2,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "examples": {
          "n": 5,
          "k50": 375,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.2,
            "50": 0.4,
            "100": 0.4,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 25,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": false,
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 178.39102073499998,
      "gpu": "NVIDIA A100-SXM4-80GB",
      "source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "plan_sha256": "5044ef7fc62847030786c27ed2e806a8ead98d38227de136e6363ac9d3d47a6d",
      "protocol_sha256": "41bf0c46a6ee5a34b51797ac9aa431cc028dff2ebd96ad2399fea27103a300d7",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen35-9b": {
      "complete": true,
      "model": "Qwen/Qwen3.5-9B",
      "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 44,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "proprietà emergenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "proprietà emergenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "proprietà emergenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Proprietà emergenti",
              "correct": false
            }
          ],
          "logprob": -16.519245147705078,
          "logprob_with_eos": -16.53377914428711,
          "rank": {
            "rank": 27,
            "rank_pessimistic": 27,
            "near_tie_rank_interval": [
              27,
              27
            ]
          },
          "prefix_rank": {
            "rank": 27,
            "rank_pessimistic": 27,
            "near_tie_rank_interval": [
              27,
              27
            ]
          },
          "numerical_check": {
            "cached_single": [
              -16.598539352416992,
              -16.582595825195312,
              -0.015943074598908424
            ],
            "uncached_single": [
              -16.5336856842041,
              -16.517925262451172,
              -0.01576237939298153
            ],
            "cached_batch": [
              -16.53377914428711,
              -16.519245147705078,
              -0.014533373527228832
            ],
            "cache_vs_full_max_error": 0.06485366821289062
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 42,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "NATURE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Nuclear",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "NATURE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "NATURE",
              "correct": false
            }
          ],
          "logprob": -22.651594161987305,
          "logprob_with_eos": -22.672962188720703,
          "rank": {
            "rank": 7,
            "rank_pessimistic": 7,
            "near_tie_rank_interval": [
              7,
              8
            ]
          },
          "prefix_rank": {
            "rank": 8,
            "rank_pessimistic": 8,
            "near_tie_rank_interval": [
              7,
              8
            ]
          },
          "numerical_check": {
            "cached_single": [
              -22.473594665527344,
              -22.452404022216797,
              -0.02119101770222187
            ],
            "uncached_single": [
              -22.627914428710938,
              -22.60654640197754,
              -0.02136758342385292
            ],
            "cached_batch": [
              -22.672962188720703,
              -22.651594161987305,
              -0.021368984133005142
            ],
            "cache_vs_full_max_error": 0.15431976318359375
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 47,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Palline di fagioli in teglia",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Palafitte",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "99",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Palafitte (9 9)",
              "correct": false
            }
          ],
          "logprob": -22.985748291015625,
          "logprob_with_eos": -23.11911392211914,
          "rank": {
            "rank": 273,
            "rank_pessimistic": 273,
            "near_tie_rank_interval": [
              268,
              275
            ]
          },
          "prefix_rank": {
            "rank": 298,
            "rank_pessimistic": 298,
            "near_tie_rank_interval": [
              290,
              300
            ]
          },
          "numerical_check": {
            "cached_single": [
              -23.179683685302734,
              -23.044876098632812,
              -0.13480953872203827
            ],
            "uncached_single": [
              -23.19518280029297,
              -23.072996139526367,
              -0.12218766659498215
            ],
            "cached_batch": [
              -23.11911392211914,
              -22.985748291015625,
              -0.1333656758069992
            ],
            "cache_vs_full_max_error": 0.028120040893554688
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 49,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "NON POSSO PARLARE (5 4)",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Non posso parlare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "NON POSSO PARLARE (5 4)",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Non posso parlare (5 4).",
              "correct": false
            }
          ],
          "logprob": -23.222698211669922,
          "logprob_with_eos": -23.277629852294922,
          "rank": {
            "rank": 240,
            "rank_pessimistic": 240,
            "near_tie_rank_interval": [
              236,
              243
            ]
          },
          "prefix_rank": {
            "rank": 242,
            "rank_pessimistic": 242,
            "near_tie_rank_interval": [
              239,
              246
            ]
          },
          "numerical_check": {
            "cached_single": [
              -23.189186096191406,
              -23.13467788696289,
              -0.05451053008437157
            ],
            "uncached_single": [
              -23.31509017944336,
              -23.26080322265625,
              -0.054288122802972794
            ],
            "cached_batch": [
              -23.277629852294922,
              -23.222698211669922,
              -0.05493173003196716
            ],
            "cache_vs_full_max_error": 0.12612533569335938
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 64,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "UNO DUE TRE QUATTRO CINQUE SEI SETTE OTTO NINE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "1989",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "UNO DUE TRE SEI NINE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "UNO, SEI, NINE",
              "correct": false
            }
          ],
          "logprob": -23.739028930664062,
          "logprob_with_eos": -24.075895309448242,
          "rank": {
            "rank": 270,
            "rank_pessimistic": 270,
            "near_tie_rank_interval": [
              267,
              275
            ]
          },
          "prefix_rank": {
            "rank": 280,
            "rank_pessimistic": 280,
            "near_tie_rank_interval": [
              279,
              282
            ]
          },
          "numerical_check": {
            "cached_single": [
              -24.131980895996094,
              -23.792625427246094,
              -0.3393542170524597
            ],
            "uncached_single": [
              -24.14580726623535,
              -23.81157684326172,
              -0.3342309594154358
            ],
            "cached_batch": [
              -24.075895309448242,
              -23.739028930664062,
              -0.33686596155166626
            ],
            "cache_vs_full_max_error": 0.018951416015625
          }
        },
        {
          "condition": "examples",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 1769,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "proprietà emergenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "proprietà emergenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "proprietà emergenti",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "tutto per il meglio",
              "correct": false
            }
          ],
          "logprob": -14.770344734191895,
          "logprob_with_eos": -15.042333602905273,
          "rank": {
            "rank": 191,
            "rank_pessimistic": 191,
            "near_tie_rank_interval": [
              189,
              197
            ]
          },
          "prefix_rank": {
            "rank": 178,
            "rank_pessimistic": 178,
            "near_tie_rank_interval": [
              177,
              179
            ]
          },
          "numerical_check": {
            "cached_single": [
              -14.99038028717041,
              -14.71778392791748,
              -0.2725972533226013
            ],
            "uncached_single": [
              -15.0463285446167,
              -14.774772644042969,
              -0.27155572175979614
            ],
            "cached_batch": [
              -15.042333602905273,
              -14.770344734191895,
              -0.27198874950408936
            ],
            "cache_vs_full_max_error": 0.05698871612548828
          }
        },
        {
          "condition": "examples",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 1767,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "alpha particle",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "lead and radon",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "fourteen",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "uranium-238",
              "correct": false
            }
          ],
          "logprob": -10.787069320678711,
          "logprob_with_eos": -10.801329612731934,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.815598487854004,
              -10.801581382751465,
              -0.014017109759151936
            ],
            "uncached_single": [
              -10.792072296142578,
              -10.776323318481445,
              -0.01574806310236454
            ],
            "cached_batch": [
              -10.801329612731934,
              -10.787069320678711,
              -0.014260996133089066
            ],
            "cache_vs_full_max_error": 0.02525806427001953
          }
        },
        {
          "condition": "examples",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 1772,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "piano di lavoro",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "proprietà commutativa della somma",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "palazzi aerei",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "piano di attacco",
              "correct": false
            }
          ],
          "logprob": -15.962020874023438,
          "logprob_with_eos": -16.009002685546875,
          "rank": {
            "rank": 236,
            "rank_pessimistic": 236,
            "near_tie_rank_interval": [
              235,
              240
            ]
          },
          "prefix_rank": {
            "rank": 241,
            "rank_pessimistic": 241,
            "near_tie_rank_interval": [
              240,
              245
            ]
          },
          "numerical_check": {
            "cached_single": [
              -16.040752410888672,
              -15.995015144348145,
              -0.04573631286621094
            ],
            "uncached_single": [
              -16.014968872070312,
              -15.968374252319336,
              -0.046594828367233276
            ],
            "cached_batch": [
              -16.009002685546875,
              -15.962020874023438,
              -0.04698343202471733
            ],
            "cache_vs_full_max_error": 0.026640892028808594
          }
        },
        {
          "condition": "examples",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 1774,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "non posso parlare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "non posso parlare",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "ho il bavaglio",
              "correct": false
            }
          ],
          "logprob": -13.172876358032227,
          "logprob_with_eos": -13.208749771118164,
          "rank": {
            "rank": 65,
            "rank_pessimistic": 65,
            "near_tie_rank_interval": [
              64,
              68
            ]
          },
          "prefix_rank": {
            "rank": 68,
            "rank_pessimistic": 68,
            "near_tie_rank_interval": [
              67,
              72
            ]
          },
          "numerical_check": {
            "cached_single": [
              -13.287680625915527,
              -13.252132415771484,
              -0.0355483703315258
            ],
            "uncached_single": [
              -13.324782371520996,
              -13.288366317749023,
              -0.03641590103507042
            ],
            "cached_batch": [
              -13.208749771118164,
              -13.172876358032227,
              -0.03587470203638077
            ],
            "cache_vs_full_max_error": 0.03710174560546875
          }
        },
        {
          "condition": "examples",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 1789,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "una donna orribile manipola l'opinione pubblica",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "un'opera di propaganda",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "una donna orribile manipola l'opinione pubblica",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "trattativa con le donne",
              "correct": false
            }
          ],
          "logprob": -15.203144073486328,
          "logprob_with_eos": -15.332315444946289,
          "rank": {
            "rank": 154,
            "rank_pessimistic": 154,
            "near_tie_rank_interval": [
              150,
              156
            ]
          },
          "prefix_rank": {
            "rank": 161,
            "rank_pessimistic": 161,
            "near_tie_rank_interval": [
              158,
              165
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.434757232666016,
              -15.306671142578125,
              -0.12808635830879211
            ],
            "uncached_single": [
              -15.49886417388916,
              -15.371177673339844,
              -0.12768679857254028
            ],
            "cached_batch": [
              -15.332315444946289,
              -15.203144073486328,
              -0.12917135655879974
            ],
            "cache_vs_full_max_error": 0.06450653076171875
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 152,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Società",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Fenomeno",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "INNOVAZIONE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SOLUZIONE",
              "correct": false
            }
          ],
          "logprob": -16.78460693359375,
          "logprob_with_eos": -16.793560028076172,
          "rank": {
            "rank": 56,
            "rank_pessimistic": 56,
            "near_tie_rank_interval": [
              56,
              56
            ]
          },
          "prefix_rank": {
            "rank": 56,
            "rank_pessimistic": 56,
            "near_tie_rank_interval": [
              56,
              56
            ]
          },
          "numerical_check": {
            "cached_single": [
              -17.052316665649414,
              -17.043628692626953,
              -0.008688503876328468
            ],
            "uncached_single": [
              -16.870641708374023,
              -16.861675262451172,
              -0.008965947665274143
            ],
            "cached_batch": [
              -16.793560028076172,
              -16.78460693359375,
              -0.008951178751885891
            ],
            "cache_vs_full_max_error": 0.18195343017578125
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 150,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Radioactive",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "radioactive",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Radioactive",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Nuclear Power",
              "correct": false
            }
          ],
          "logprob": -14.341643333435059,
          "logprob_with_eos": -14.349905014038086,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -14.374990463256836,
              -14.366619110107422,
              -0.008371027186512947
            ],
            "uncached_single": [
              -14.351314544677734,
              -14.343036651611328,
              -0.008278341963887215
            ],
            "cached_batch": [
              -14.349905014038086,
              -14.341643333435059,
              -0.008261908777058125
            ],
            "cache_vs_full_max_error": 0.023675918579101562
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 155,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "SALVATAGGIO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FONDI DI BASSO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "PALAZZO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SOLUZIONE:",
              "correct": false
            }
          ],
          "logprob": -23.285675048828125,
          "logprob_with_eos": -23.30044174194336,
          "rank": {
            "rank": 406,
            "rank_pessimistic": 406,
            "near_tie_rank_interval": [
              402,
              412
            ]
          },
          "prefix_rank": {
            "rank": 412,
            "rank_pessimistic": 412,
            "near_tie_rank_interval": [
              407,
              416
            ]
          },
          "numerical_check": {
            "cached_single": [
              -23.264734268188477,
              -23.249841690063477,
              -0.014892591163516045
            ],
            "uncached_single": [
              -23.262292861938477,
              -23.247390747070312,
              -0.014901164919137955
            ],
            "cached_batch": [
              -23.30044174194336,
              -23.285675048828125,
              -0.014767267741262913
            ],
            "cache_vs_full_max_error": 0.0024509429931640625
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 157,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "SILENZIO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "SILENZIOSA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "IN CAPOLUOGO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SILENZIO",
              "correct": false
            }
          ],
          "logprob": -20.78806495666504,
          "logprob_with_eos": -20.794979095458984,
          "rank": {
            "rank": 120,
            "rank_pessimistic": 120,
            "near_tie_rank_interval": [
              120,
              121
            ]
          },
          "prefix_rank": {
            "rank": 122,
            "rank_pessimistic": 122,
            "near_tie_rank_interval": [
              120,
              122
            ]
          },
          "numerical_check": {
            "cached_single": [
              -21.000225067138672,
              -20.993579864501953,
              -0.006644891574978828
            ],
            "uncached_single": [
              -20.879894256591797,
              -20.872838973999023,
              -0.007055013906210661
            ],
            "cached_batch": [
              -20.794979095458984,
              -20.78806495666504,
              -0.006915089208632708
            ],
            "cache_vs_full_max_error": 0.12074089050292969
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 172,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "CASA DI VETRO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FEMMINA IN UNO ZERO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "CASA 609",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "UNA CINA IN",
              "correct": false
            }
          ],
          "logprob": -17.884395599365234,
          "logprob_with_eos": -17.93583106994629,
          "rank": {
            "rank": 56,
            "rank_pessimistic": 56,
            "near_tie_rank_interval": [
              55,
              57
            ]
          },
          "prefix_rank": {
            "rank": 60,
            "rank_pessimistic": 60,
            "near_tie_rank_interval": [
              60,
              61
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.024059295654297,
              -17.9720458984375,
              -0.052013445645570755
            ],
            "uncached_single": [
              -17.953811645507812,
              -17.901012420654297,
              -0.05279998853802681
            ],
            "cached_batch": [
              -17.93583106994629,
              -17.884395599365234,
              -0.05143589898943901
            ],
            "cache_vs_full_max_error": 0.07103347778320312
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 240,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.0,
            "5": 0.0,
            "10": 0.2,
            "20": 0.2,
            "50": 0.4,
            "100": 0.4,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "examples": {
          "n": 5,
          "k50": 154,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.2,
            "50": 0.2,
            "100": 0.4,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 56,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.2,
            "50": 0.2,
            "100": 0.6,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": false,
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 332.63778365,
      "gpu": "NVIDIA A100-SXM4-80GB",
      "source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "plan_sha256": "5044ef7fc62847030786c27ed2e806a8ead98d38227de136e6363ac9d3d47a6d",
      "protocol_sha256": "41bf0c46a6ee5a34b51797ac9aa431cc028dff2ebd96ad2399fea27103a300d7",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen35-27b": {
      "complete": true,
      "model": "Qwen/Qwen3.5-27B",
      "revision": "fc05daec18b0a78c049392ed2e771dde82bdf654",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 44,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "SISTEMICITÀ",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Sistemica",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "autocoscienza",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SINERGIA",
              "correct": false
            }
          ],
          "logprob": -16.3880672454834,
          "logprob_with_eos": -16.4127140045166,
          "rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              25,
              25
            ]
          },
          "prefix_rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              25,
              25
            ]
          },
          "numerical_check": {
            "cached_single": [
              -16.30373191833496,
              -16.27927017211914,
              -0.024460773915052414
            ],
            "uncached_single": [
              -16.33204460144043,
              -16.30779457092285,
              -0.024250661954283714
            ],
            "cached_batch": [
              -16.4127140045166,
              -16.3880672454834,
              -0.024646533653140068
            ],
            "cache_vs_full_max_error": 0.028524398803710938
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 42,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "RADIUM, THORIUM SERIES",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "THORON RADIUM",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "RADIUM, THORIUM SERIES",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "ACTINIUM SERIES",
              "correct": false
            }
          ],
          "logprob": -15.580482482910156,
          "logprob_with_eos": -15.601449966430664,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.50532341003418,
              -15.481980323791504,
              -0.02334316074848175
            ],
            "uncached_single": [
              -15.543380737304688,
              -15.522836685180664,
              -0.02054411917924881
            ],
            "cached_batch": [
              -15.601449966430664,
              -15.580482482910156,
              -0.02096761018037796
            ],
            "cache_vs_full_max_error": 0.040856361389160156
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 47,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "PALLAVOLO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FONDAZIONE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "PALAZZI",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "PALAFITTE",
              "correct": false
            }
          ],
          "logprob": -22.656391143798828,
          "logprob_with_eos": -22.777015686035156,
          "rank": {
            "rank": 251,
            "rank_pessimistic": 251,
            "near_tie_rank_interval": [
              248,
              255
            ]
          },
          "prefix_rank": {
            "rank": 268,
            "rank_pessimistic": 268,
            "near_tie_rank_interval": [
              261,
              272
            ]
          },
          "numerical_check": {
            "cached_single": [
              -22.72927474975586,
              -22.609832763671875,
              -0.11944042146205902
            ],
            "uncached_single": [
              -22.672378540039062,
              -22.553382873535156,
              -0.1189941018819809
            ],
            "cached_batch": [
              -22.777015686035156,
              -22.656391143798828,
              -0.12062518298625946
            ],
            "cache_vs_full_max_error": 0.056896209716796875
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 49,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "NON POSSO PARLARE (5 4) = NON POSSO PARLARE (5 4)",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "NON POSSO PARLARE (5 4) = NON POSSO PARLARE (5 4)",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "NON POSSO PARLARE (5 4) = NON POSSO PARLARE (5 4)",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SILENZIO",
              "correct": false
            }
          ],
          "logprob": -19.06884765625,
          "logprob_with_eos": -19.076358795166016,
          "rank": {
            "rank": 64,
            "rank_pessimistic": 64,
            "near_tie_rank_interval": [
              64,
              66
            ]
          },
          "prefix_rank": {
            "rank": 65,
            "rank_pessimistic": 65,
            "near_tie_rank_interval": [
              64,
              66
            ]
          },
          "numerical_check": {
            "cached_single": [
              -19.114500045776367,
              -19.106178283691406,
              -0.008322085253894329
            ],
            "uncached_single": [
              -19.08006477355957,
              -19.072614669799805,
              -0.007450413890182972
            ],
            "cached_batch": [
              -19.076358795166016,
              -19.06884765625,
              -0.007510875817388296
            ],
            "cache_vs_full_max_error": 0.034435272216796875
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 64,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "MADONNA DI CASCIA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FEMMINA POLITICA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "MARIA TERESA D'ASBURGO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "UNO SCACCHIERE",
              "correct": false
            }
          ],
          "logprob": -18.621612548828125,
          "logprob_with_eos": -18.82891845703125,
          "rank": {
            "rank": 13,
            "rank_pessimistic": 13,
            "near_tie_rank_interval": [
              12,
              13
            ]
          },
          "prefix_rank": {
            "rank": 14,
            "rank_pessimistic": 14,
            "near_tie_rank_interval": [
              14,
              14
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.829561233520508,
              -18.623929977416992,
              -0.20563118159770966
            ],
            "uncached_single": [
              -18.96127700805664,
              -18.733375549316406,
              -0.22789840400218964
            ],
            "cached_batch": [
              -18.82891845703125,
              -18.621612548828125,
              -0.20730635523796082
            ],
            "cache_vs_full_max_error": 0.1317157745361328
          }
        },
        {
          "condition": "examples",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 1769,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "caratteristiche del tutto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "qualità del tutto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "sintesi",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "tutti i sensi",
              "correct": false
            }
          ],
          "logprob": -15.753989219665527,
          "logprob_with_eos": -15.818645477294922,
          "rank": {
            "rank": 156,
            "rank_pessimistic": 156,
            "near_tie_rank_interval": [
              154,
              160
            ]
          },
          "prefix_rank": {
            "rank": 156,
            "rank_pessimistic": 156,
            "near_tie_rank_interval": [
              153,
              159
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.870558738708496,
              -15.80499267578125,
              -0.06556589156389236
            ],
            "uncached_single": [
              -15.87653636932373,
              -15.811656951904297,
              -0.06487882137298584
            ],
            "cached_batch": [
              -15.818645477294922,
              -15.753989219665527,
              -0.06465613842010498
            ],
            "cache_vs_full_max_error": 0.006664276123046875
          }
        },
        {
          "condition": "examples",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 1767,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "half life of uranium",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "radioactive chain",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "half-life of nature",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Thorium and Helium",
              "correct": false
            }
          ],
          "logprob": -9.726333618164062,
          "logprob_with_eos": -9.845285415649414,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -9.927001953125,
              -9.808597564697266,
              -0.1184050589799881
            ],
            "uncached_single": [
              -9.843984603881836,
              -9.724860191345215,
              -0.11912395805120468
            ],
            "cached_batch": [
              -9.845285415649414,
              -9.726333618164062,
              -0.1189521923661232
            ],
            "cache_vs_full_max_error": 0.08373737335205078
          }
        },
        {
          "condition": "examples",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 1772,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "fondamenta in acqua",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "casa sull'acqua",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "fondamenta in acqua",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "fondamenta in acqua",
              "correct": false
            }
          ],
          "logprob": -22.09870719909668,
          "logprob_with_eos": -22.152517318725586,
          "rank": {
            "rank": 621,
            "rank_pessimistic": 621,
            "near_tie_rank_interval": [
              614,
              628
            ]
          },
          "prefix_rank": {
            "rank": 639,
            "rank_pessimistic": 639,
            "near_tie_rank_interval": [
              631,
              645
            ]
          },
          "numerical_check": {
            "cached_single": [
              -22.239303588867188,
              -22.18154525756836,
              -0.05776096135377884
            ],
            "uncached_single": [
              -22.176593780517578,
              -22.1237850189209,
              -0.05280858278274536
            ],
            "cached_batch": [
              -22.152517318725586,
              -22.09870719909668,
              -0.05381017178297043
            ],
            "cache_vs_full_max_error": 0.06270980834960938
          }
        },
        {
          "condition": "examples",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 1774,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "muto per legge",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto di guerra",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto e sordo",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "silenzio assoluto",
              "correct": false
            }
          ],
          "logprob": -12.427600860595703,
          "logprob_with_eos": -12.442232131958008,
          "rank": {
            "rank": 15,
            "rank_pessimistic": 15,
            "near_tie_rank_interval": [
              15,
              15
            ]
          },
          "prefix_rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              16,
              16
            ]
          },
          "numerical_check": {
            "cached_single": [
              -12.309690475463867,
              -12.296483039855957,
              -0.013207333162426949
            ],
            "uncached_single": [
              -12.474485397338867,
              -12.461041450500488,
              -0.013444124720990658
            ],
            "cached_batch": [
              -12.442232131958008,
              -12.427600860595703,
              -0.014631355181336403
            ],
            "cache_vs_full_max_error": 0.164794921875
          }
        },
        {
          "condition": "examples",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 1789,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "la signora delle menzogne",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "La signora manipola",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "la stampa è la prima",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "malvagia manipolatrice",
              "correct": false
            }
          ],
          "logprob": -11.910810470581055,
          "logprob_with_eos": -11.97558307647705,
          "rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              15,
              16
            ]
          },
          "prefix_rank": {
            "rank": 17,
            "rank_pessimistic": 17,
            "near_tie_rank_interval": [
              17,
              17
            ]
          },
          "numerical_check": {
            "cached_single": [
              -11.941633224487305,
              -11.882478713989258,
              -0.0591542087495327
            ],
            "uncached_single": [
              -11.974143981933594,
              -11.91623592376709,
              -0.05790756270289421
            ],
            "cached_batch": [
              -11.97558307647705,
              -11.910810470581055,
              -0.06477200984954834
            ],
            "cache_vs_full_max_error": 0.03375720977783203
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 152,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "SISTEMA SOLARE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Fatti di stato",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Caratteristiche",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "VANTAGGI NATURALI",
              "correct": false
            }
          ],
          "logprob": -15.698328018188477,
          "logprob_with_eos": -17.633071899414062,
          "rank": {
            "rank": 184,
            "rank_pessimistic": 184,
            "near_tie_rank_interval": [
              183,
              185
            ]
          },
          "prefix_rank": {
            "rank": 104,
            "rank_pessimistic": 104,
            "near_tie_rank_interval": [
              102,
              106
            ]
          },
          "numerical_check": {
            "cached_single": [
              -17.624858856201172,
              -15.639749526977539,
              -1.9851089715957642
            ],
            "uncached_single": [
              -17.6607608795166,
              -15.73283576965332,
              -1.9279264211654663
            ],
            "cached_batch": [
              -17.633071899414062,
              -15.698328018188477,
              -1.9347443580627441
            ],
            "cache_vs_full_max_error": 0.09308624267578125
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 150,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "ACTINIDE SERIES",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "ONE HALF LIFE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "ONE ACTOR",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "ACTINIDE SERIES",
              "correct": false
            }
          ],
          "logprob": -15.512200355529785,
          "logprob_with_eos": -15.513860702514648,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.460108757019043,
              -15.458470344543457,
              -0.0016388569492846727
            ],
            "uncached_single": [
              -15.480560302734375,
              -15.478917121887207,
              -0.0016436175210401416
            ],
            "cached_batch": [
              -15.513860702514648,
              -15.512200355529785,
              -0.0016599221853539348
            ],
            "cache_vs_full_max_error": 0.02045154571533203
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 155,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "PALAZZO DI VETRO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "PALAZZO IN LEGNO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "PALAZZO IN LEGNO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "CASE SULL'ACQUA",
              "correct": false
            }
          ],
          "logprob": -23.98583221435547,
          "logprob_with_eos": -24.012723922729492,
          "rank": {
            "rank": 297,
            "rank_pessimistic": 297,
            "near_tie_rank_interval": [
              291,
              301
            ]
          },
          "prefix_rank": {
            "rank": 330,
            "rank_pessimistic": 330,
            "near_tie_rank_interval": [
              322,
              334
            ]
          },
          "numerical_check": {
            "cached_single": [
              -23.86404800415039,
              -23.837339401245117,
              -0.026707347482442856
            ],
            "uncached_single": [
              -24.027307510375977,
              -23.996774673461914,
              -0.03053269162774086
            ],
            "cached_batch": [
              -24.012723922729492,
              -23.98583221435547,
              -0.02689303830265999
            ],
            "cache_vs_full_max_error": 0.16325950622558594
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 157,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "MUTO SILENZIO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "SILENZIO D'ORO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "MUTO SOGNO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SORDO MUTO",
              "correct": false
            }
          ],
          "logprob": -18.739919662475586,
          "logprob_with_eos": -18.743785858154297,
          "rank": {
            "rank": 18,
            "rank_pessimistic": 18,
            "near_tie_rank_interval": [
              18,
              19
            ]
          },
          "prefix_rank": {
            "rank": 18,
            "rank_pessimistic": 18,
            "near_tie_rank_interval": [
              18,
              19
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.75835609436035,
              -18.75446319580078,
              -0.003894009394571185
            ],
            "uncached_single": [
              -18.713598251342773,
              -18.709774017333984,
              -0.0038244216702878475
            ],
            "cached_batch": [
              -18.743785858154297,
              -18.739919662475586,
              -0.0038664599414914846
            ],
            "cache_vs_full_max_error": 0.044757843017578125
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 172,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "UNA FALSA PROFEZIA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FALSA PROFEZIA POLITICA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "FAKE NEWS FEMMINILE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "UNA CATTIVA LINGUA",
              "correct": false
            }
          ],
          "logprob": -15.216556549072266,
          "logprob_with_eos": -15.338756561279297,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              3
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.442331314086914,
              -15.322553634643555,
              -0.11977719515562057
            ],
            "uncached_single": [
              -15.433361053466797,
              -15.311779022216797,
              -0.12158266454935074
            ],
            "cached_batch": [
              -15.338756561279297,
              -15.216556549072266,
              -0.12220074981451035
            ],
            "cache_vs_full_max_error": 0.010774612426757812
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 25,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.4,
            "50": 0.6,
            "100": 0.8,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "examples": {
          "n": 5,
          "k50": 16,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 18,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": false,
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 500.063098389,
      "gpu": "NVIDIA H100 80GB HBM3",
      "source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "plan_sha256": "5044ef7fc62847030786c27ed2e806a8ead98d38227de136e6363ac9d3d47a6d",
      "protocol_sha256": "41bf0c46a6ee5a34b51797ac9aa431cc028dff2ebd96ad2399fea27103a300d7",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    },
    "qwen38-27b": {
      "complete": true,
      "model": "Qwen/Qwen3.8-27B",
      "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "records": [
        {
          "condition": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 44,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "COMPLESSITÀ",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FONOFONO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "INTELLIGENZA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "COMPLESSITÀ",
              "correct": false
            }
          ],
          "logprob": -18.176382064819336,
          "logprob_with_eos": -18.17667007446289,
          "rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              16,
              16
            ]
          },
          "prefix_rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              16,
              16
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.009763717651367,
              -18.009475708007812,
              -0.0002862997353076935
            ],
            "uncached_single": [
              -18.070777893066406,
              -18.070480346679688,
              -0.0002980979916173965
            ],
            "cached_batch": [
              -18.17667007446289,
              -18.176382064819336,
              -0.000288087350782007
            ],
            "cache_vs_full_max_error": 0.06101417541503906
          }
        },
        {
          "condition": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 42,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "ALPHA DECAY",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "ALPHA DECAY",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "ALPHA DECAY",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "ACTINIDE SERIES",
              "correct": false
            }
          ],
          "logprob": -19.916934967041016,
          "logprob_with_eos": -19.917491912841797,
          "rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "prefix_rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "numerical_check": {
            "cached_single": [
              -19.850753784179688,
              -19.85019302368164,
              -0.0005629861843772233
            ],
            "uncached_single": [
              -19.83483123779297,
              -19.83427619934082,
              -0.0005548844928853214
            ],
            "cached_batch": [
              -19.917491912841797,
              -19.916934967041016,
              -0.0005569098866544664
            ],
            "cache_vs_full_max_error": 0.01592254638671875
          }
        },
        {
          "condition": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 47,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "CASA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "CASTELLO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "BATTICUORE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "CASE",
              "correct": false
            }
          ],
          "logprob": -25.412425994873047,
          "logprob_with_eos": -25.418079376220703,
          "rank": {
            "rank": 327,
            "rank_pessimistic": 327,
            "near_tie_rank_interval": [
              322,
              330
            ]
          },
          "prefix_rank": {
            "rank": 326,
            "rank_pessimistic": 326,
            "near_tie_rank_interval": [
              322,
              330
            ]
          },
          "numerical_check": {
            "cached_single": [
              -25.435279846191406,
              -25.429473876953125,
              -0.005805772263556719
            ],
            "uncached_single": [
              -25.39859962463379,
              -25.39284896850586,
              -0.005751726217567921
            ],
            "cached_batch": [
              -25.418079376220703,
              -25.412425994873047,
              -0.005654411856085062
            ],
            "cache_vs_full_max_error": 0.03668022155761719
          }
        },
        {
          "condition": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 49,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "SILENZIO ASSOLUTO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "SILENZIO ASSOLUTO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "NON POSSO PARLARE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SILENZIO ASSOLUTO",
              "correct": false
            }
          ],
          "logprob": -22.993408203125,
          "logprob_with_eos": -22.993446350097656,
          "rank": {
            "rank": 156,
            "rank_pessimistic": 156,
            "near_tie_rank_interval": [
              154,
              156
            ]
          },
          "prefix_rank": {
            "rank": 156,
            "rank_pessimistic": 156,
            "near_tie_rank_interval": [
              154,
              156
            ]
          },
          "numerical_check": {
            "cached_single": [
              -22.96186065673828,
              -22.96181869506836,
              -4.184158387943171e-05
            ],
            "uncached_single": [
              -23.012218475341797,
              -23.012176513671875,
              -4.1960789531003684e-05
            ],
            "cached_batch": [
              -22.993446350097656,
              -22.993408203125,
              -3.8265450712060556e-05
            ],
            "cache_vs_full_max_error": 0.050357818603515625
          }
        },
        {
          "condition": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 64,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "UNA DONNA ORRIBILE MANIPOLA L'OPINIONE PUBBLICA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "UNO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "UNA DONNA ORRIBILE MANIPOLA L'OPINIONE PUBBLICA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "UNA DONNA ORRIBILE MANIPOLA L'OPINIONE PUBBLICA",
              "correct": false
            }
          ],
          "logprob": -21.20359992980957,
          "logprob_with_eos": -21.207691192626953,
          "rank": {
            "rank": 19,
            "rank_pessimistic": 19,
            "near_tie_rank_interval": [
              18,
              20
            ]
          },
          "prefix_rank": {
            "rank": 20,
            "rank_pessimistic": 20,
            "near_tie_rank_interval": [
              19,
              20
            ]
          },
          "numerical_check": {
            "cached_single": [
              -21.09766387939453,
              -21.09351921081543,
              -0.004146194085478783
            ],
            "uncached_single": [
              -21.056110382080078,
              -21.051485061645508,
              -0.004625929053872824
            ],
            "cached_batch": [
              -21.207691192626953,
              -21.20359992980957,
              -0.004092770162969828
            ],
            "cache_vs_full_max_error": 0.042034149169921875
          }
        },
        {
          "condition": "examples",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 1897,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "emergere",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "",
              "correct": false
            }
          ],
          "logprob": -18.01232147216797,
          "logprob_with_eos": -18.046184539794922,
          "rank": {
            "rank": 287,
            "rank_pessimistic": 287,
            "near_tie_rank_interval": [
              284,
              290
            ]
          },
          "prefix_rank": {
            "rank": 286,
            "rank_pessimistic": 286,
            "near_tie_rank_interval": [
              284,
              287
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.05548667907715,
              -18.02118492126465,
              -0.034302450716495514
            ],
            "uncached_single": [
              -18.14558219909668,
              -18.11124610900879,
              -0.03433539345860481
            ],
            "cached_batch": [
              -18.046184539794922,
              -18.01232147216797,
              -0.033862560987472534
            ],
            "cache_vs_full_max_error": 0.09009552001953125
          }
        },
        {
          "condition": "examples",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 1895,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "Half life",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "radioactive decay",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "Radium 226",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "Thorium Radium",
              "correct": false
            }
          ],
          "logprob": -10.307794570922852,
          "logprob_with_eos": -10.318931579589844,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -10.310306549072266,
              -10.299577713012695,
              -0.01072841975837946
            ],
            "uncached_single": [
              -10.377765655517578,
              -10.366729736328125,
              -0.011036896146833897
            ],
            "cached_batch": [
              -10.318931579589844,
              -10.307794570922852,
              -0.011138049885630608
            ],
            "cache_vs_full_max_error": 0.0674591064453125
          }
        },
        {
          "condition": "examples",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 1900,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "abitazioni su pali",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "case in legno",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "casette in acqua",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "palo del portone",
              "correct": false
            }
          ],
          "logprob": -19.110374450683594,
          "logprob_with_eos": -19.138885498046875,
          "rank": {
            "rank": 480,
            "rank_pessimistic": 480,
            "near_tie_rank_interval": [
              476,
              485
            ]
          },
          "prefix_rank": {
            "rank": 480,
            "rank_pessimistic": 480,
            "near_tie_rank_interval": [
              476,
              486
            ]
          },
          "numerical_check": {
            "cached_single": [
              -19.180456161499023,
              -19.151947021484375,
              -0.028509993106126785
            ],
            "uncached_single": [
              -19.10383415222168,
              -19.07489013671875,
              -0.02894541062414646
            ],
            "cached_batch": [
              -19.138885498046875,
              -19.110374450683594,
              -0.028509877622127533
            ],
            "cache_vs_full_max_error": 0.077056884765625
          }
        },
        {
          "condition": "examples",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 1902,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "muto assoluto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "muto in scena",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "muto assoluto",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "",
              "correct": false
            }
          ],
          "logprob": -13.584449768066406,
          "logprob_with_eos": -13.58492374420166,
          "rank": {
            "rank": 59,
            "rank_pessimistic": 59,
            "near_tie_rank_interval": [
              56,
              59
            ]
          },
          "prefix_rank": {
            "rank": 59,
            "rank_pessimistic": 59,
            "near_tie_rank_interval": [
              56,
              59
            ]
          },
          "numerical_check": {
            "cached_single": [
              -13.467004776000977,
              -13.466540336608887,
              -0.0004646890447475016
            ],
            "uncached_single": [
              -13.430359840393066,
              -13.429892539978027,
              -0.00046695294440723956
            ],
            "cached_batch": [
              -13.58492374420166,
              -13.584449768066406,
              -0.0004737447015941143
            ],
            "cache_vs_full_max_error": 0.036647796630859375
          }
        },
        {
          "condition": "examples",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 1917,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "la stampa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "\"La bella e la bestia\"",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "la stampa",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "femmina manipolatrice",
              "correct": false
            }
          ],
          "logprob": -11.201459884643555,
          "logprob_with_eos": -11.204025268554688,
          "rank": {
            "rank": 26,
            "rank_pessimistic": 26,
            "near_tie_rank_interval": [
              24,
              26
            ]
          },
          "prefix_rank": {
            "rank": 26,
            "rank_pessimistic": 26,
            "near_tie_rank_interval": [
              25,
              26
            ]
          },
          "numerical_check": {
            "cached_single": [
              -11.260042190551758,
              -11.257772445678711,
              -0.0022699085529893637
            ],
            "uncached_single": [
              -11.255977630615234,
              -11.253674507141113,
              -0.0023023784160614014
            ],
            "cached_batch": [
              -11.204025268554688,
              -11.201459884643555,
              -0.0025659038219600916
            ],
            "cache_vs_full_max_error": 0.004097938537597656
          }
        },
        {
          "condition": "definition",
          "case_id": "palafitte",
          "answer": "palafitte",
          "prompt_tokens": 152,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "INTELLIGENZA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "Fenomenologia",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "INATTESURA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "AUTOCOSTRUZIONE",
              "correct": false
            }
          ],
          "logprob": -20.004005432128906,
          "logprob_with_eos": -20.007314682006836,
          "rank": {
            "rank": 141,
            "rank_pessimistic": 141,
            "near_tie_rank_interval": [
              140,
              143
            ]
          },
          "prefix_rank": {
            "rank": 141,
            "rank_pessimistic": 141,
            "near_tie_rank_interval": [
              140,
              143
            ]
          },
          "numerical_check": {
            "cached_single": [
              -20.015865325927734,
              -20.012577056884766,
              -0.0032883400563150644
            ],
            "uncached_single": [
              -20.136919021606445,
              -20.133602142333984,
              -0.003316499525681138
            ],
            "cached_batch": [
              -20.007314682006836,
              -20.004005432128906,
              -0.0033086577896028757
            ],
            "cache_vs_full_max_error": 0.12105369567871094
          }
        },
        {
          "condition": "definition",
          "case_id": "uranium",
          "answer": "lead generation",
          "prompt_tokens": 150,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "NATURE'S URANIUM",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "THOR NITRATE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "NATURE'S OWN",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "ACTUALLY FORTUNATE",
              "correct": false
            }
          ],
          "logprob": -15.088068962097168,
          "logprob_with_eos": -15.088447570800781,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "numerical_check": {
            "cached_single": [
              -15.23077392578125,
              -15.230449676513672,
              -0.000325388420606032
            ],
            "uncached_single": [
              -15.098943710327148,
              -15.098577499389648,
              -0.00036554806865751743
            ],
            "cached_batch": [
              -15.088447570800781,
              -15.088068962097168,
              -0.0003781795676331967
            ],
            "cache_vs_full_max_error": 0.13187217712402344
          }
        },
        {
          "condition": "definition",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "prompt_tokens": 155,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "CASA SU PALI",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "CASE SULLE ACQUE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "CASA SU PALI",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "CASE SULL'ACQUA",
              "correct": false
            }
          ],
          "logprob": -24.120519638061523,
          "logprob_with_eos": -24.12290382385254,
          "rank": {
            "rank": 362,
            "rank_pessimistic": 362,
            "near_tie_rank_interval": [
              357,
              364
            ]
          },
          "prefix_rank": {
            "rank": 364,
            "rank_pessimistic": 364,
            "near_tie_rank_interval": [
              359,
              368
            ]
          },
          "numerical_check": {
            "cached_single": [
              -24.15694808959961,
              -24.154552459716797,
              -0.002394667826592922
            ],
            "uncached_single": [
              -24.221492767333984,
              -24.219131469726562,
              -0.002363271312788129
            ],
            "cached_batch": [
              -24.12290382385254,
              -24.120519638061523,
              -0.002385153668001294
            ],
            "cache_vs_full_max_error": 0.06457901000976562
          }
        },
        {
          "condition": "definition",
          "case_id": "speech",
          "answer": "servo muto",
          "prompt_tokens": 157,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "MUTO SILENZIO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "MUTO COM",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "MUTO SILENZIO",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "SILENZA",
              "correct": false
            }
          ],
          "logprob": -17.995372772216797,
          "logprob_with_eos": -17.995407104492188,
          "rank": {
            "rank": 13,
            "rank_pessimistic": 13,
            "near_tie_rank_interval": [
              13,
              13
            ]
          },
          "prefix_rank": {
            "rank": 13,
            "rank_pessimistic": 13,
            "near_tie_rank_interval": [
              13,
              13
            ]
          },
          "numerical_check": {
            "cached_single": [
              -18.063146591186523,
              -18.063108444213867,
              -3.814624506048858e-05
            ],
            "uncached_single": [
              -18.1146240234375,
              -18.114585876464844,
              -3.85038583772257e-05
            ],
            "cached_batch": [
              -17.995407104492188,
              -17.995372772216797,
              -3.480850500636734e-05
            ],
            "cache_vs_full_max_error": 0.05147743225097656
          }
        },
        {
          "condition": "definition",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "prompt_tokens": 172,
          "generations": [
            {
              "greedy": true,
              "seed": 0,
              "output": "FATTA A PANE",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 17,
              "output": "FACCIA DI RAME",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 29,
              "output": "BELLA DONNA",
              "correct": false
            },
            {
              "greedy": false,
              "seed": 43,
              "output": "MALE FAMA PREGIUDIZIO",
              "correct": false
            }
          ],
          "logprob": -16.20012092590332,
          "logprob_with_eos": -16.20317840576172,
          "rank": {
            "rank": 7,
            "rank_pessimistic": 7,
            "near_tie_rank_interval": [
              7,
              7
            ]
          },
          "prefix_rank": {
            "rank": 7,
            "rank_pessimistic": 7,
            "near_tie_rank_interval": [
              7,
              7
            ]
          },
          "numerical_check": {
            "cached_single": [
              -16.090206146240234,
              -16.087135314941406,
              -0.003070166567340493
            ],
            "uncached_single": [
              -16.185659408569336,
              -16.182737350463867,
              -0.0029229081701487303
            ],
            "cached_batch": [
              -16.20317840576172,
              -16.20012092590332,
              -0.003057569032534957
            ],
            "cache_vs_full_max_error": 0.09560203552246094
          }
        }
      ],
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 19,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.0,
            "5": 0.2,
            "10": 0.2,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "examples": {
          "n": 5,
          "k50": 59,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.2,
            "50": 0.4,
            "100": 0.6,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        },
        "definition": {
          "n": 5,
          "k50": 13,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.4,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 1.0
          },
          "greedy_correct": 0,
          "sample_correct": 0
        }
      },
      "training_performed": false,
      "adapters_loaded": false,
      "cache_pristine_verified": true,
      "batch_size": 8,
      "seconds": 407.06897982600003,
      "gpu": "NVIDIA H100 80GB HBM3",
      "source_sha256": "646e11bef3805189a12eaa2d3d02a7fbcaaba65324c1d8c5898d246117448465",
      "plan_sha256": "5044ef7fc62847030786c27ed2e806a8ead98d38227de136e6363ac9d3d47a6d",
      "protocol_sha256": "41bf0c46a6ee5a34b51797ac9aa431cc028dff2ebd96ad2399fea27103a300d7",
      "packages": {
        "torch": "2.9.1",
        "transformers": "5.19.0",
        "peft": "0.21.2"
      }
    }
  },
  "lora_reference": {
    "qwen3-4b": {
      "model": "Qwen/Qwen3-4B-Instruct-2507",
      "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 126,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.0,
            "5": 0.0,
            "10": 0.0,
            "20": 0.2,
            "50": 0.2,
            "100": 0.2,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 126,
            "recall_at_k": {
              "1": 0.0,
              "3": 0.0,
              "5": 0.0,
              "10": 0.0,
              "20": 0.0,
              "50": 0.0,
              "100": 0.0,
              "500": 1.0
            }
          }
        },
        "v2_s17": {
          "n": 5,
          "k50": 40,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "italian_only": {
            "n": 4,
            "k50": 40,
            "recall_at_k": {
              "1": 0.25,
              "3": 0.25,
              "5": 0.25,
              "10": 0.25,
              "20": 0.25,
              "50": 0.5,
              "100": 0.5,
              "500": 0.75
            }
          }
        },
        "v2_s29": {
          "n": 5,
          "k50": 31,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.4,
            "50": 0.6,
            "100": 0.8,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 31,
            "recall_at_k": {
              "1": 0.25,
              "3": 0.25,
              "5": 0.25,
              "10": 0.25,
              "20": 0.25,
              "50": 0.5,
              "100": 0.75,
              "500": 1.0
            }
          }
        }
      },
      "records": [
        {
          "variant": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -26.95734405517578,
          "logprob": -26.957223892211914,
          "rank": {
            "rank": 126,
            "rank_pessimistic": 126,
            "near_tie_rank_interval": [
              124,
              127
            ]
          },
          "prefix_rank": {
            "rank": 133,
            "rank_pessimistic": 133,
            "near_tie_rank_interval": [
              131,
              136
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -37.58454895019531,
          "logprob": -37.58454513549805,
          "rank": {
            "rank": 14,
            "rank_pessimistic": 14,
            "near_tie_rank_interval": [
              14,
              14
            ]
          },
          "prefix_rank": {
            "rank": 14,
            "rank_pessimistic": 14,
            "near_tie_rank_interval": [
              14,
              14
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -29.97042465209961,
          "logprob": -29.968585968017578,
          "rank": {
            "rank": 183,
            "rank_pessimistic": 183,
            "near_tie_rank_interval": [
              181,
              184
            ]
          },
          "prefix_rank": {
            "rank": 189,
            "rank_pessimistic": 189,
            "near_tie_rank_interval": [
              188,
              190
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -29.0720157623291,
          "logprob": -29.069719314575195,
          "rank": {
            "rank": 126,
            "rank_pessimistic": 126,
            "near_tie_rank_interval": [
              125,
              128
            ]
          },
          "prefix_rank": {
            "rank": 132,
            "rank_pessimistic": 132,
            "near_tie_rank_interval": [
              131,
              135
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -29.048362731933594,
          "logprob": -29.04578399658203,
          "rank": {
            "rank": 262,
            "rank_pessimistic": 262,
            "near_tie_rank_interval": [
              258,
              264
            ]
          },
          "prefix_rank": {
            "rank": 287,
            "rank_pessimistic": 287,
            "near_tie_rank_interval": [
              284,
              290
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -21.341026306152344,
          "logprob": -13.225709915161133,
          "rank": {
            "rank": 673,
            "rank_pessimistic": 673,
            "near_tie_rank_interval": [
              667,
              681
            ]
          },
          "prefix_rank": {
            "rank": 40,
            "rank_pessimistic": 40,
            "near_tie_rank_interval": [
              40,
              43
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -10.47758960723877,
          "logprob": -10.473047256469727,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -18.796485900878906,
          "logprob": -18.795791625976562,
          "rank": {
            "rank": 291,
            "rank_pessimistic": 291,
            "near_tie_rank_interval": [
              289,
              296
            ]
          },
          "prefix_rank": {
            "rank": 295,
            "rank_pessimistic": 295,
            "near_tie_rank_interval": [
              292,
              298
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -12.560924530029297,
          "logprob": -12.560426712036133,
          "rank": {
            "rank": 40,
            "rank_pessimistic": 40,
            "near_tie_rank_interval": [
              39,
              40
            ]
          },
          "prefix_rank": {
            "rank": 40,
            "rank_pessimistic": 40,
            "near_tie_rank_interval": [
              39,
              40
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -8.54166316986084,
          "logprob": -8.510786056518555,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          }
        },
        {
          "variant": "v2_s29",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -14.175491333007812,
          "logprob": -13.032413482666016,
          "rank": {
            "rank": 62,
            "rank_pessimistic": 62,
            "near_tie_rank_interval": [
              61,
              62
            ]
          },
          "prefix_rank": {
            "rank": 38,
            "rank_pessimistic": 38,
            "near_tie_rank_interval": [
              38,
              40
            ]
          }
        },
        {
          "variant": "v2_s29",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -10.936697006225586,
          "logprob": -10.928658485412598,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        },
        {
          "variant": "v2_s29",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -19.347047805786133,
          "logprob": -19.345928192138672,
          "rank": {
            "rank": 364,
            "rank_pessimistic": 364,
            "near_tie_rank_interval": [
              361,
              366
            ]
          },
          "prefix_rank": {
            "rank": 370,
            "rank_pessimistic": 370,
            "near_tie_rank_interval": [
              367,
              371
            ]
          }
        },
        {
          "variant": "v2_s29",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -12.051965713500977,
          "logprob": -12.05176067352295,
          "rank": {
            "rank": 31,
            "rank_pessimistic": 31,
            "near_tie_rank_interval": [
              30,
              31
            ]
          },
          "prefix_rank": {
            "rank": 31,
            "rank_pessimistic": 31,
            "near_tie_rank_interval": [
              30,
              31
            ]
          }
        },
        {
          "variant": "v2_s29",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -7.785784721374512,
          "logprob": -7.745201110839844,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          }
        }
      ]
    },
    "qwen35-9b": {
      "model": "Qwen/Qwen3.5-9B",
      "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 240,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.0,
            "5": 0.0,
            "10": 0.2,
            "20": 0.2,
            "50": 0.4,
            "100": 0.4,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 240,
            "recall_at_k": {
              "1": 0.0,
              "3": 0.0,
              "5": 0.0,
              "10": 0.0,
              "20": 0.0,
              "50": 0.25,
              "100": 0.25,
              "500": 1.0
            }
          }
        },
        "v2_s17": {
          "n": 5,
          "k50": 15,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.4,
            "5": 0.4,
            "10": 0.4,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 0.8
          },
          "italian_only": {
            "n": 4,
            "k50": 15,
            "recall_at_k": {
              "1": 0.0,
              "3": 0.25,
              "5": 0.25,
              "10": 0.25,
              "20": 0.5,
              "50": 0.5,
              "100": 0.5,
              "500": 0.75
            }
          }
        }
      },
      "records": [
        {
          "variant": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -16.49067497253418,
          "logprob": -16.474700927734375,
          "rank": {
            "rank": 26,
            "rank_pessimistic": 26,
            "near_tie_rank_interval": [
              26,
              27
            ]
          },
          "prefix_rank": {
            "rank": 26,
            "rank_pessimistic": 26,
            "near_tie_rank_interval": [
              26,
              27
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -22.495704650878906,
          "logprob": -22.474292755126953,
          "rank": {
            "rank": 7,
            "rank_pessimistic": 7,
            "near_tie_rank_interval": [
              7,
              7
            ]
          },
          "prefix_rank": {
            "rank": 7,
            "rank_pessimistic": 7,
            "near_tie_rank_interval": [
              7,
              7
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -23.142627716064453,
          "logprob": -23.020069122314453,
          "rank": {
            "rank": 275,
            "rank_pessimistic": 275,
            "near_tie_rank_interval": [
              270,
              279
            ]
          },
          "prefix_rank": {
            "rank": 301,
            "rank_pessimistic": 301,
            "near_tie_rank_interval": [
              296,
              302
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -23.32718276977539,
          "logprob": -23.27265739440918,
          "rank": {
            "rank": 240,
            "rank_pessimistic": 240,
            "near_tie_rank_interval": [
              238,
              245
            ]
          },
          "prefix_rank": {
            "rank": 242,
            "rank_pessimistic": 242,
            "near_tie_rank_interval": [
              238,
              246
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -24.0216064453125,
          "logprob": -23.685760498046875,
          "rank": {
            "rank": 260,
            "rank_pessimistic": 260,
            "near_tie_rank_interval": [
              258,
              266
            ]
          },
          "prefix_rank": {
            "rank": 274,
            "rank_pessimistic": 274,
            "near_tie_rank_interval": [
              270,
              277
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -22.84039306640625,
          "logprob": -16.552038192749023,
          "rank": {
            "rank": 870,
            "rank_pessimistic": 870,
            "near_tie_rank_interval": [
              858,
              877
            ]
          },
          "prefix_rank": {
            "rank": 250,
            "rank_pessimistic": 250,
            "near_tie_rank_interval": [
              247,
              255
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -12.546905517578125,
          "logprob": -11.737702369689941,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -15.255881309509277,
          "logprob": -15.255824089050293,
          "rank": {
            "rank": 147,
            "rank_pessimistic": 147,
            "near_tie_rank_interval": [
              147,
              150
            ]
          },
          "prefix_rank": {
            "rank": 150,
            "rank_pessimistic": 150,
            "near_tie_rank_interval": [
              149,
              154
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -9.851633071899414,
          "logprob": -9.847380638122559,
          "rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          },
          "prefix_rank": {
            "rank": 2,
            "rank_pessimistic": 2,
            "near_tie_rank_interval": [
              2,
              2
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -11.952330589294434,
          "logprob": -10.991887092590332,
          "rank": {
            "rank": 15,
            "rank_pessimistic": 15,
            "near_tie_rank_interval": [
              14,
              15
            ]
          },
          "prefix_rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              25,
              27
            ]
          }
        }
      ]
    },
    "qwen35-27b": {
      "model": "Qwen/Qwen3.5-27B",
      "revision": "fc05daec18b0a78c049392ed2e771dde82bdf654",
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 25,
          "recall_at_k": {
            "1": 0.2,
            "3": 0.2,
            "5": 0.2,
            "10": 0.2,
            "20": 0.4,
            "50": 0.6,
            "100": 0.8,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 25,
            "recall_at_k": {
              "1": 0.0,
              "3": 0.0,
              "5": 0.0,
              "10": 0.0,
              "20": 0.25,
              "50": 0.5,
              "100": 0.75,
              "500": 1.0
            }
          }
        },
        "v2_s17": {
          "n": 5,
          "k50": 4,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.4,
            "5": 0.6,
            "10": 0.6,
            "20": 0.8,
            "50": 0.8,
            "100": 1.0,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 4,
            "recall_at_k": {
              "1": 0.25,
              "3": 0.25,
              "5": 0.5,
              "10": 0.5,
              "20": 0.75,
              "50": 0.75,
              "100": 1.0,
              "500": 1.0
            }
          }
        }
      },
      "records": [
        {
          "variant": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -16.454511642456055,
          "logprob": -16.427536010742188,
          "rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              24,
              25
            ]
          },
          "prefix_rank": {
            "rank": 25,
            "rank_pessimistic": 25,
            "near_tie_rank_interval": [
              24,
              25
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -15.469335556030273,
          "logprob": -15.44831657409668,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -22.661109924316406,
          "logprob": -22.540220260620117,
          "rank": {
            "rank": 243,
            "rank_pessimistic": 243,
            "near_tie_rank_interval": [
              239,
              246
            ]
          },
          "prefix_rank": {
            "rank": 252,
            "rank_pessimistic": 252,
            "near_tie_rank_interval": [
              247,
              256
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -19.063154220581055,
          "logprob": -19.05660629272461,
          "rank": {
            "rank": 67,
            "rank_pessimistic": 67,
            "near_tie_rank_interval": [
              66,
              70
            ]
          },
          "prefix_rank": {
            "rank": 68,
            "rank_pessimistic": 68,
            "near_tie_rank_interval": [
              66,
              70
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -19.030519485473633,
          "logprob": -18.825532913208008,
          "rank": {
            "rank": 14,
            "rank_pessimistic": 14,
            "near_tie_rank_interval": [
              14,
              15
            ]
          },
          "prefix_rank": {
            "rank": 15,
            "rank_pessimistic": 15,
            "near_tie_rank_interval": [
              14,
              15
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -15.292559623718262,
          "logprob": -15.292503356933594,
          "rank": {
            "rank": 54,
            "rank_pessimistic": 54,
            "near_tie_rank_interval": [
              54,
              57
            ]
          },
          "prefix_rank": {
            "rank": 54,
            "rank_pessimistic": 54,
            "near_tie_rank_interval": [
              54,
              57
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -8.478635787963867,
          "logprob": -8.477994918823242,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -12.448873519897461,
          "logprob": -12.448799133300781,
          "rank": {
            "rank": 17,
            "rank_pessimistic": 17,
            "near_tie_rank_interval": [
              17,
              19
            ]
          },
          "prefix_rank": {
            "rank": 18,
            "rank_pessimistic": 18,
            "near_tie_rank_interval": [
              18,
              20
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -9.461259841918945,
          "logprob": -9.461005210876465,
          "rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "prefix_rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -4.942251682281494,
          "logprob": -4.937575340270996,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        }
      ]
    },
    "qwen38-27b": {
      "model": "Qwen/Qwen3.8-27B",
      "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "aggregate": {
        "base": {
          "n": 5,
          "k50": 16,
          "recall_at_k": {
            "1": 0.0,
            "3": 0.0,
            "5": 0.2,
            "10": 0.2,
            "20": 0.6,
            "50": 0.6,
            "100": 0.6,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 16,
            "recall_at_k": {
              "1": 0.0,
              "3": 0.0,
              "5": 0.0,
              "10": 0.0,
              "20": 0.5,
              "50": 0.5,
              "100": 0.5,
              "500": 1.0
            }
          }
        },
        "v2_s17": {
          "n": 5,
          "k50": 3,
          "recall_at_k": {
            "1": 0.4,
            "3": 0.6,
            "5": 0.6,
            "10": 0.6,
            "20": 0.6,
            "50": 0.8,
            "100": 1.0,
            "500": 1.0
          },
          "italian_only": {
            "n": 4,
            "k50": 3,
            "recall_at_k": {
              "1": 0.25,
              "3": 0.5,
              "5": 0.5,
              "10": 0.5,
              "20": 0.5,
              "50": 0.75,
              "100": 1.0,
              "500": 1.0
            }
          }
        }
      },
      "records": [
        {
          "variant": "base",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -18.08507537841797,
          "logprob": -18.084821701049805,
          "rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              16,
              16
            ]
          },
          "prefix_rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              16,
              16
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -19.78717803955078,
          "logprob": -19.786609649658203,
          "rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          },
          "prefix_rank": {
            "rank": 4,
            "rank_pessimistic": 4,
            "near_tie_rank_interval": [
              4,
              4
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -25.323814392089844,
          "logprob": -25.3194580078125,
          "rank": {
            "rank": 323,
            "rank_pessimistic": 323,
            "near_tie_rank_interval": [
              322,
              326
            ]
          },
          "prefix_rank": {
            "rank": 323,
            "rank_pessimistic": 323,
            "near_tie_rank_interval": [
              322,
              327
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -22.958133697509766,
          "logprob": -22.95809555053711,
          "rank": {
            "rank": 153,
            "rank_pessimistic": 153,
            "near_tie_rank_interval": [
              151,
              154
            ]
          },
          "prefix_rank": {
            "rank": 153,
            "rank_pessimistic": 153,
            "near_tie_rank_interval": [
              152,
              154
            ]
          }
        },
        {
          "variant": "base",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -21.151508331298828,
          "logprob": -21.147476196289062,
          "rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              15,
              18
            ]
          },
          "prefix_rank": {
            "rank": 16,
            "rank_pessimistic": 16,
            "near_tie_rank_interval": [
              15,
              19
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "palafitte",
          "answer": "palafitte",
          "logprob_with_eos": -19.21225929260254,
          "logprob": -19.212244033813477,
          "rank": {
            "rank": 80,
            "rank_pessimistic": 80,
            "near_tie_rank_interval": [
              79,
              82
            ]
          },
          "prefix_rank": {
            "rank": 81,
            "rank_pessimistic": 81,
            "near_tie_rank_interval": [
              79,
              82
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "uranium",
          "answer": "lead generation",
          "logprob_with_eos": -7.354955673217773,
          "logprob": -7.353803634643555,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "inverse",
          "answer": "proprietà emergenti",
          "logprob_with_eos": -14.39265251159668,
          "logprob": -14.392526626586914,
          "rank": {
            "rank": 50,
            "rank_pessimistic": 50,
            "near_tie_rank_interval": [
              48,
              50
            ]
          },
          "prefix_rank": {
            "rank": 51,
            "rank_pessimistic": 51,
            "near_tie_rank_interval": [
              49,
              51
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "speech",
          "answer": "servo muto",
          "logprob_with_eos": -8.899580001831055,
          "logprob": -8.898720741271973,
          "rank": {
            "rank": 3,
            "rank_pessimistic": 3,
            "near_tie_rank_interval": [
              3,
              3
            ]
          },
          "prefix_rank": {
            "rank": 3,
            "rank_pessimistic": 3,
            "near_tie_rank_interval": [
              3,
              3
            ]
          }
        },
        {
          "variant": "v2_s17",
          "case_id": "influenza",
          "answer": "una brutta influenza",
          "logprob_with_eos": -5.511753559112549,
          "logprob": -5.504765033721924,
          "rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          },
          "prefix_rank": {
            "rank": 1,
            "rank_pessimistic": 1,
            "near_tie_rank_interval": [
              1,
              1
            ]
          }
        }
      ]
    }
  },
  "cases": [
    {
      "id": "palafitte",
      "answer": "palafitte",
      "prompt": "Risolvi questa crittografia mnemonica italiana: proprietà emergenti (9).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    },
    {
      "id": "uranium",
      "answer": "lead generation",
      "prompt": "Solve this English mnemonic cryptogram: Natural Uranium Decay (4, 10).\nWrite only the solution, without preamble or explanation."
    },
    {
      "id": "inverse",
      "answer": "proprietà emergenti",
      "prompt": "Risolvi questa crittografia mnemonica italiana: PALAFITTE (9 9).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    },
    {
      "id": "speech",
      "answer": "servo muto",
      "prompt": "Risolvi questa crittografia mnemonica italiana: NON POSSO PARLARE (5 4).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    },
    {
      "id": "influenza",
      "answer": "una brutta influenza",
      "prompt": "Risolvi questa crittografia mnemonica italiana: UNA DONNA ORRIBILE MANIPOLA L'OPINIONE PUBBLICA (3; 6; 9).\nScrivi soltanto la soluzione, senza premesse né spiegazioni."
    }
  ],
  "example_ids": [
    "bei-p04-47",
    "bei-p28-02",
    "bei-p26-09",
    "bei-p25-06",
    "bei-p22-07",
    "bei-p25-12",
    "bei-p12-44",
    "bei-p12-24",
    "bei-p11-45",
    "bei-p06-48",
    "bei-p28-27",
    "bei-p19-14",
    "bei-p36-36",
    "bei-p12-38",
    "bei-p10-49",
    "bei-p26-13",
    "bei-p21-44",
    "bei-p18-25",
    "bei-p18-09",
    "bei-p32-47",
    "bei-p41-15",
    "bei-p40-43",
    "bei-p23-50",
    "bei-p37-24",
    "bei-p24-38",
    "bei-p11-11",
    "bei-p27-16",
    "bei-p06-17",
    "bei-p13-19",
    "bei-p06-13",
    "bei-p41-23",
    "bei-p22-30"
  ],
  "definition": "Una crittografia mnemonica è un gioco enigmistico basato sul doppio senso. Si interpreta il significato dell’esposto per trovare una parola o espressione che lo descriva e che abbia anche un’altra lettura autonoma e riconoscibile. Il passaggio è semantico: non richiede di cifrare le lettere, assegnare loro numeri o riordinarle in un anagramma. Il diagramma numerico indica le lunghezze delle parole della soluzione. La risposta deve rispettare il diagramma e sostenere entrambe le letture.",
  "candidate_count": 2008,
  "corpus_sha256": "db186802a034531f8b5cc39ba6eb980fea8216565920fa7cfa87c5d43589c7fc",
  "v2_test": [
    {
      "run": "v2-qwen3-4b-s17",
      "model": "Qwen/Qwen3-4B-Instruct-2507",
      "seed": 17,
      "test": {
        "n": 64,
        "families": 63,
        "mean_delta": 24.66868321597576,
        "median_delta": 23.3289155960083,
        "geometric_ratio": 51697911233.71974,
        "improved": 64,
        "ci95": [
          22.72212073020637,
          26.6707382465899
        ],
        "before_mean": -41.84238991141319,
        "after_mean": -17.17370669543743,
        "before_correct": 0,
        "after_correct": 0,
        "before_accepted": 0,
        "after_accepted": 0
      }
    },
    {
      "run": "v2-qwen3-4b-s29",
      "model": "Qwen/Qwen3-4B-Instruct-2507",
      "seed": 29,
      "test": {
        "n": 64,
        "families": 63,
        "mean_delta": 24.65070030838251,
        "median_delta": 23.80634880065918,
        "geometric_ratio": 50776541753.96583,
        "improved": 64,
        "ci95": [
          22.733049802668393,
          26.664147182509655
        ],
        "before_mean": -41.84238991141319,
        "after_mean": -17.19168960303068,
        "before_correct": 0,
        "after_correct": 0,
        "before_accepted": 0,
        "after_accepted": 0
      }
    },
    {
      "run": "v2-qwen35-9b-s17",
      "model": "Qwen/Qwen3.5-9B",
      "seed": 17,
      "test": {
        "n": 64,
        "families": 63,
        "mean_delta": 13.632384926080704,
        "median_delta": 14.482941627502441,
        "geometric_ratio": 832663.5400767771,
        "improved": 64,
        "ci95": [
          12.421651535278711,
          14.811209569555341
        ],
        "before_mean": -28.936178982257843,
        "after_mean": -15.30379405617714,
        "before_correct": 0,
        "after_correct": 0,
        "before_accepted": 0,
        "after_accepted": 0
      }
    },
    {
      "run": "v2-qwen35-27b-s17",
      "model": "Qwen/Qwen3.5-27B",
      "seed": 17,
      "test": {
        "n": 64,
        "families": 63,
        "mean_delta": 15.95603135600686,
        "median_delta": 15.898442506790161,
        "geometric_ratio": 8503865.272220248,
        "improved": 64,
        "ci95": [
          14.623429809108613,
          17.285757690947502
        ],
        "before_mean": -27.169612005352974,
        "after_mean": -11.213580649346113,
        "before_correct": 0,
        "after_correct": 1,
        "before_accepted": 0,
        "after_accepted": 1
      }
    },
    {
      "run": "v2-qwen38-27b-s17",
      "model": "Qwen/Qwen3.8-27B",
      "seed": 17,
      "test": {
        "n": 64,
        "families": 63,
        "mean_delta": 18.382463678717613,
        "median_delta": 18.3638973236084,
        "geometric_ratio": 96250399.205927,
        "improved": 64,
        "ci95": [
          16.75752618664787,
          19.93703996239023
        ],
        "before_mean": -30.22445134818554,
        "after_mean": -11.841987669467926,
        "before_correct": 0,
        "after_correct": 0,
        "before_accepted": 0,
        "after_accepted": 0
      }
    }
  ]
}